imports: spack uses importlib instead of imp when available

- `imp` is deprecated and seems to have started having some weird
  issues on certain Linux versions.
  - In particular, the file argument to `load_source` is ignored on
    arch linux with Python 3.7.

- `imp` is the only way to do imports in 2.6, so we'll keep it around for
  now and use it if importlib won't work.

- `importlib` is the new import system, and it allows us to get
  lower-level access to the import implementation.

- This consolidates all import logic into `spack.util.imp`, and make it
  use `importlib` if it's avialable.

lib/spack/spack/compilers/__init__.py

@@ -26,7 +26,6 @@
 system and configuring Spack to use multiple compilers.
 """
 import os
-import imp
 
 from llnl.util.lang import list_modules
 
@@ -35,7 +34,7 @@
 import spack.spec
 import spack.config
 import spack.architecture
-
+import spack.util.imp as simp
 from spack.util.naming import mod_to_class
 
 _imported_compilers_module = 'spack.compilers'
@@ -361,7 +360,7 @@ def class_for_compiler_name(compiler_name):
     assert(supported(compiler_name))
 
     file_path = os.path.join(spack.paths.compilers_path, compiler_name + ".py")
-    compiler_mod = imp.load_source(_imported_compilers_module, file_path)
+    compiler_mod = simp.load_source(_imported_compilers_module, file_path)
     cls = getattr(compiler_mod, mod_to_class(compiler_name))
 
     # make a note of the name in the module so we can get to it easily.

lib/spack/spack/directives.py

@@ -48,7 +48,6 @@ class OpenMpi(Package):
 
 import collections
 import functools
-import inspect
 import os.path
 import re
 from six import string_types
@@ -115,11 +114,11 @@ def __new__(cls, name, bases, attr_dict):
     def __init__(cls, name, bases, attr_dict):
         # The class is being created: if it is a package we must ensure
         # that the directives are called on the class to set it up
-        module = inspect.getmodule(cls)
-        if 'spack.pkg' in module.__name__:
+
+        if 'spack.pkg' in cls.__module__:
             # Package name as taken
             # from llnl.util.lang.get_calling_module_name
-            pkg_name = module.__name__.split('.')[-1]
+            pkg_name = cls.__module__.split('.')[-1]
             setattr(cls, 'name', pkg_name)
 
             # Ensure the presence of the dictionaries associated

lib/spack/spack/hooks/__init__.py

@@ -41,10 +41,10 @@
    systems (e.g. modules, dotkit, etc.) or to add other custom
    features.
 """
-import imp
 import os.path
 
 import spack.paths
+import spack.util.imp as simp
 from llnl.util.lang import memoized, list_modules
 
 
@@ -54,7 +54,7 @@ def all_hook_modules():
     for name in list_modules(spack.paths.hooks_path):
         mod_name = __name__ + '.' + name
         path = os.path.join(spack.paths.hooks_path, name) + ".py"
-        mod = imp.load_source(mod_name, path)
+        mod = simp.load_source(mod_name, path)
         modules.append(mod)
 
     return modules

lib/spack/spack/repo.py

@@ -29,10 +29,8 @@
 import errno
 import sys
 import inspect
-import imp
 import re
 import traceback
-import tempfile
 import json
 from contextlib import contextmanager
 from six import string_types
@@ -54,11 +52,13 @@
 import spack.caches
 import spack.error
 import spack.spec
+import spack.util.imp as simp
 from spack.provider_index import ProviderIndex
 from spack.util.path import canonicalize_path
 from spack.util.naming import NamespaceTrie, valid_module_name
 from spack.util.naming import mod_to_class, possible_spack_module_names
 
+
 #: Super-namespace for all packages.
 #: Package modules are imported as spack.pkg.<namespace>.<pkg-name>.
 repo_namespace     = 'spack.pkg'
@@ -994,9 +994,8 @@ def _get_pkg_module(self, pkg_name):
             fullname = "%s.%s" % (self.full_namespace, pkg_name)
 
             try:
-                with import_lock():
-                    with prepend_open(file_path, text=_package_prepend) as f:
-                        module = imp.load_source(fullname, file_path, f)
+                module = simp.load_source(fullname, file_path,
+                                          prepend=_package_prepend)
             except SyntaxError as e:
                 # SyntaxError strips the path from the filename so we need to
                 # manually construct the error message in order to give the
@@ -1146,31 +1145,6 @@ def set_path(repo):
     return append
 
 
-@contextmanager
-def import_lock():
-    imp.acquire_lock()
-    yield
-    imp.release_lock()
-
-
-@contextmanager
-def prepend_open(f, *args, **kwargs):
-    """Open a file for reading, but prepend with some text prepended
-
-    Arguments are same as for ``open()``, with one keyword argument,
-    ``text``, specifying the text to prepend.
-    """
-    text = kwargs.get('text', None)
-
-    with open(f, *args) as f:
-        with tempfile.NamedTemporaryFile(mode='w+') as tf:
-            if text:
-                tf.write(text + '\n')
-            tf.write(f.read())
-            tf.seek(0)
-            yield tf.file
-
-
 @contextmanager
 def swap(repo_path):
     """Temporarily use another RepoPath."""

lib/spack/spack/util/imp/__init__.py

@@ -0,0 +1,41 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Consolidated module for all imports done by Spack.
+
+Many parts of Spack have to import Python code. This utility package
+wraps Spack's interface with Python's import system.
+
+We do this because Python's import system is confusing and changes from
+Python version to Python version, and we should be able to adapt our
+approach to the underlying implementation.
+
+Currently, this uses ``importlib.machinery`` where available and ``imp``
+when ``importlib`` is not completely usable.
+"""
+
+try:
+    from .importlib_importer import load_source  # noqa
+except ImportError:
+    from .imp_importer import load_source        # noqa

lib/spack/spack/util/imp/imp_importer.py

@@ -0,0 +1,86 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Implementation of Spack imports that uses imp underneath.
+
+``imp`` is deprecated in newer versions of Python, but is the only option
+in Python 2.6.
+"""
+import imp
+import tempfile
+from contextlib import contextmanager
+
+
+@contextmanager
+def import_lock():
+    imp.acquire_lock()
+    yield
+    imp.release_lock()
+
+
+def load_source(full_name, path, prepend=None):
+    """Import a Python module from source.
+
+    Load the source file and add it to ``sys.modules``.
+
+    Args:
+        full_name (str): full name of the module to be loaded
+        path (str): path to the file that should be loaded
+        prepend (str, optional): some optional code to prepend to the
+            loaded module; e.g., can be used to inject import statements
+
+    Returns:
+        (ModuleType): the loaded module
+    """
+    with import_lock():
+        if prepend is None:
+            return imp.load_source(full_name, path)
+        else:
+            with prepend_open(path, text=prepend) as f:
+                return imp.load_source(full_name, path, f)
+
+
+@contextmanager
+def prepend_open(f, *args, **kwargs):
+    """Open a file for reading, but prepend with some text prepended
+
+    Arguments are same as for ``open()``, with one keyword argument,
+    ``text``, specifying the text to prepend.
+
+    We have to write and read a tempfile for the ``imp``-based importer,
+    as the ``file`` argument to ``imp.load_source()`` requires a
+    low-level file handle.
+
+    See the ``importlib``-based importer for a faster way to do this in
+    later versions of python.
+    """
+    text = kwargs.get('text', None)
+
+    with open(f, *args) as f:
+        with tempfile.NamedTemporaryFile(mode='w+') as tf:
+            if text:
+                tf.write(text + '\n')
+            tf.write(f.read())
+            tf.seek(0)
+            yield tf.file

lib/spack/spack/util/imp/importlib_importer.py

@@ -0,0 +1,61 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""Implementation of Spack imports that uses importlib underneath.
+
+``importlib`` is only fully implemented in Python 3.
+"""
+from importlib.machinery import SourceFileLoader
+
+
+class PrependFileLoader(SourceFileLoader):
+    def __init__(self, full_name, path, prepend=None):
+        super(PrependFileLoader, self).__init__(full_name, path)
+        self.prepend = prepend
+
+    def get_data(self, path):
+        data = super(PrependFileLoader, self).get_data(path)
+        if path != self.path or self.prepend is None:
+            return data
+        else:
+            return self.prepend.encode() + b"\n" + data
+
+
+def load_source(full_name, path, prepend=None):
+    """Import a Python module from source.
+
+    Load the source file and add it to ``sys.modules``.
+
+    Args:
+        full_name (str): full name of the module to be loaded
+        path (str): path to the file that should be loaded
+        prepend (str, optional): some optional code to prepend to the
+            loaded module; e.g., can be used to inject import statements
+
+    Returns:
+        (ModuleType): the loaded module
+    """
+    # use our custom loader
+    loader = PrependFileLoader(full_name, path, prepend)
+    return loader.load_module()