Windows rpath support (#31930)

Add a post-install step which runs (only) on Windows to modify an
install prefix, adding symlinks to all dependency libraries.

Windows does not have the same concept of RPATHs as Linux, but when
resolving symbols will check the local directory for dependency
libraries; by placing a symlink to each dependency library in the
directory with the library that needs it, the package can then
use all Spack-built dependencies.

Note:

* This collects dependency libraries based on Package.rpath, which
  includes only direct link dependencies
* There is no examination of libraries to check what dependencies
  they require, so all libraries of dependencies are symlinked
  into any directory of the package which contains libraries

lib/spack/llnl/util/filesystem.py

@@ -22,7 +22,7 @@
 from llnl.util import tty
 from llnl.util.compat import Sequence
 from llnl.util.lang import dedupe, memoized
-from llnl.util.symlink import symlink
+from llnl.util.symlink import islink, symlink
 
 from spack.util.executable import Executable
 from spack.util.path import path_to_os_path, system_path_filter
@@ -637,7 +637,11 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
                 if symlinks:
                     target = os.readlink(s)
                     if os.path.isabs(target):
-                        new_target = re.sub(abs_src, abs_dest, target)
+
+                        def escaped_path(path):
+                            return path.replace("\\", r"\\")
+
+                        new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target)
                         if new_target != target:
                             tty.debug("Redirecting link {0} to {1}".format(target, new_target))
                             target = new_target
@@ -1903,7 +1907,11 @@ def names(self):
                 name = x[3:]
 
             # Valid extensions include: ['.dylib', '.so', '.a']
-            for ext in [".dylib", ".so", ".a"]:
+            # on non Windows platform
+            # Windows valid library extensions are:
+            # ['.dll', '.lib']
+            valid_exts = [".dll", ".lib"] if is_windows else [".dylib", ".so", ".a"]
+            for ext in valid_exts:
                 i = name.rfind(ext)
                 if i != -1:
                     names.append(name[:i])
@@ -2046,15 +2054,23 @@ def find_libraries(libraries, root, shared=True, recursive=False):
         message = message.format(find_libraries.__name__, type(libraries))
         raise TypeError(message)
 
+    if is_windows:
+        static = "lib"
+        shared = "dll"
+    else:
+        # Used on both Linux and macOS
+        static = "a"
+        shared = "so"
+
     # Construct the right suffix for the library
     if shared:
         # Used on both Linux and macOS
-        suffixes = ["so"]
+        suffixes = [shared]
         if sys.platform == "darwin":
             # Only used on macOS
             suffixes.append("dylib")
     else:
-        suffixes = ["a"]
+        suffixes = [static]
 
     # List of libraries we are searching with suffixes
     libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes]
@@ -2067,7 +2083,11 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     # perform first non-recursive search in root/lib then in root/lib64 and
     # finally search all of root recursively. The search stops when the first
     # match is found.
-    for subdir in ("lib", "lib64"):
+    common_lib_dirs = ["lib", "lib64"]
+    if is_windows:
+        common_lib_dirs.extend(["bin", "Lib"])
+
+    for subdir in common_lib_dirs:
         dirname = join_path(root, subdir)
         if not os.path.isdir(dirname):
             continue
@@ -2080,6 +2100,155 @@ def find_libraries(libraries, root, shared=True, recursive=False):
     return LibraryList(found_libs)
 
 
+def find_all_shared_libraries(root, recursive=False):
+    """Convenience function that returns the list of all shared libraries found
+    in the directory passed as argument.
+
+    See documentation for `llnl.util.filesystem.find_libraries` for more information
+    """
+    return find_libraries("*", root=root, shared=True, recursive=recursive)
+
+
+def find_all_static_libraries(root, recursive=False):
+    """Convenience function that returns the list of all static libraries found
+    in the directory passed as argument.
+
+    See documentation for `llnl.util.filesystem.find_libraries` for more information
+    """
+    return find_libraries("*", root=root, shared=False, recursive=recursive)
+
+
+def find_all_libraries(root, recursive=False):
+    """Convenience function that returns the list of all libraries found
+    in the directory passed as argument.
+
+    See documentation for `llnl.util.filesystem.find_libraries` for more information
+    """
+
+    return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries(
+        root, recursive=recursive
+    )
+
+
+class WindowsSimulatedRPath(object):
+    """Class representing Windows filesystem rpath analog
+
+    One instance of this class is associated with a package (only on Windows)
+    For each lib/binary directory in an associated package, this class introduces
+    a symlink to any/all dependent libraries/binaries. This includes the packages
+    own bin/lib directories, meaning the libraries are linked to the bianry directory
+    and vis versa.
+    """
+
+    def __init__(self, package, link_install_prefix=True):
+        """
+        Args:
+            package (spack.package_base.PackageBase): Package requiring links
+            link_install_prefix (bool): Link against package's own install or stage root.
+                Packages that run their own executables during build and require rpaths to
+                the build directory during build time require this option. Default: install
+                root
+        """
+        self.pkg = package
+        self._addl_rpaths = set()
+        self.link_install_prefix = link_install_prefix
+        self._internal_links = set()
+
+    @property
+    def link_dest(self):
+        """
+        Set of directories where package binaries/libraries are located.
+        """
+        if hasattr(self.pkg, "libs") and self.pkg.libs:
+            pkg_libs = set(self.pkg.libs.directories)
+        else:
+            pkg_libs = set((self.pkg.prefix.lib, self.pkg.prefix.lib64))
+
+        return pkg_libs | set([self.pkg.prefix.bin]) | self.internal_links
+
+    @property
+    def internal_links(self):
+        """
+        linking that would need to be established within the package itself. Useful for links
+        against extension modules/build time executables/internal linkage
+        """
+        return self._internal_links
+
+    def add_internal_links(self, *dest):
+        """
+        Incorporate additional paths into the rpath (sym)linking scheme.
+
+        Paths provided to this method are linked against by a package's libraries
+        and libraries found at these paths are linked against a package's binaries.
+        (i.e. /site-packages -> /bin and /bin -> /site-packages)
+
+        Specified paths should be outside of a package's lib, lib64, and bin
+        directories.
+        """
+        self._internal_links = self._internal_links | set(*dest)
+
+    @property
+    def link_targets(self):
+        """
+        Set of libraries this package needs to link against during runtime
+        These packages will each be symlinked into the packages lib and binary dir
+        """
+
+        dependent_libs = []
+        for path in self.pkg.rpath:
+            dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True)))
+        for extra_path in self._addl_rpaths:
+            dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True)))
+        return set(dependent_libs)
+
+    def include_additional_link_paths(self, *paths):
+        """
+        Add libraries found at the root of provided paths to runtime linking
+
+        These are libraries found outside of the typical scope of rpath linking
+        that require manual inclusion in a runtime linking scheme
+
+        Args:
+            *paths (str): arbitrary number of paths to be added to runtime linking
+        """
+        self._addl_rpaths = self._addl_rpaths | set(paths)
+
+    def establish_link(self):
+        """
+        (sym)link packages to runtime dependencies based on RPath configuration for
+        Windows heuristics
+        """
+        # from build_environment.py:463
+        # The top-level package is always RPATHed. It hasn't been installed yet
+        # so the RPATHs are added unconditionally
+
+        # for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
+        # install a symlink to each dependent library
+        for library, lib_dir in itertools.product(self.link_targets, self.link_dest):
+            if not path_contains_subdirectory(library, lib_dir):
+                file_name = os.path.basename(library)
+                dest_file = os.path.join(lib_dir, file_name)
+                if os.path.exists(lib_dir):
+                    try:
+                        symlink(library, dest_file)
+                    # For py2 compatibility, we have to catch the specific Windows error code
+                    # associate with trying to create a file that already exists (winerror 183)
+                    except OSError as e:
+                        if e.winerror == 183:
+                            # We have either already symlinked or we are encoutering a naming clash
+                            # either way, we don't want to overwrite existing libraries
+                            already_linked = islink(dest_file)
+                            tty.debug(
+                                "Linking library %s to %s failed, " % (library, dest_file)
+                                + "already linked."
+                                if already_linked
+                                else "library with name %s already exists." % file_name
+                            )
+                            pass
+                        else:
+                            raise e
+
+
 @system_path_filter
 @memoized
 def can_access_dir(path):

lib/spack/spack/installer.py

@@ -84,6 +84,9 @@
 #: queue invariants).
 STATUS_REMOVED = "removed"
 
+is_windows = sys.platform == "win32"
+is_osx = sys.platform == "darwin"
+
 
 class InstallAction(object):
     #: Don't perform an install
@@ -165,7 +168,9 @@ def _do_fake_install(pkg):
     if not pkg.name.startswith("lib"):
         library = "lib" + library
 
-    dso_suffix = ".dylib" if sys.platform == "darwin" else ".so"
+    plat_shared = ".dll" if is_windows else ".so"
+    plat_static = ".lib" if is_windows else ".a"
+    dso_suffix = ".dylib" if is_osx else plat_shared
 
     # Install fake command
     fs.mkdirp(pkg.prefix.bin)
@@ -180,7 +185,7 @@ def _do_fake_install(pkg):
 
     # Install fake shared and static libraries
     fs.mkdirp(pkg.prefix.lib)
-    for suffix in [dso_suffix, ".a"]:
+    for suffix in [dso_suffix, plat_static]:
         fs.touch(os.path.join(pkg.prefix.lib, library + suffix))
 
     # Install fake man page
@@ -1214,7 +1219,10 @@ def _install_task(self, task):
             spack.package_base.PackageBase._verbose = spack.build_environment.start_build_process(
                 pkg, build_process, install_args
             )
-
+            # Currently this is how RPATH-like behavior is achieved on Windows, after install
+            # establish runtime linkage via Windows Runtime link object
+            # Note: this is a no-op on non Windows platforms
+            pkg.windows_establish_runtime_linkage()
             # Note: PARENT of the build process adds the new package to
             # the database, so that we don't need to re-read from file.
             spack.store.db.add(pkg.spec, spack.store.layout, explicit=explicit)

lib/spack/spack/package_base.py

@@ -97,6 +97,9 @@
 _spack_configure_argsfile = "spack-configure-args.txt"
 
 
+is_windows = sys.platform == "win32"
+
+
 def preferred_version(pkg):
     """
     Returns a sorted list of the preferred versions of the package.
@@ -182,6 +185,30 @@ def copy(self):
             return other
 
 
+class WindowsRPathMeta(object):
+    """Collection of functionality surrounding Windows RPATH specific features
+
+    This is essentially meaningless for all other platforms
+    due to their use of RPATH. All methods within this class are no-ops on
+    non Windows. Packages can customize and manipulate this class as
+    they would a genuine RPATH, i.e. adding directories that contain
+    runtime library dependencies"""
+
+    def add_search_paths(self, *path):
+        """Add additional rpaths that are not implicitly included in the search
+        scheme
+        """
+        self.win_rpath.include_additional_link_paths(*path)
+
+    def windows_establish_runtime_linkage(self):
+        """Establish RPATH on Windows
+
+        Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
+        """
+        if is_windows:
+            self.win_rpath.establish_link()
+
+
 #: Registers which are the detectable packages, by repo and package name
 #: Need a pass of package repositories to be filled.
 detectable_packages = collections.defaultdict(list)
@@ -221,7 +248,7 @@ def to_windows_exe(exe):
                 plat_exe = []
                 if hasattr(cls, "executables"):
                     for exe in cls.executables:
-                        if sys.platform == "win32":
+                        if is_windows:
                             exe = to_windows_exe(exe)
                         plat_exe.append(exe)
                 return plat_exe
@@ -513,7 +540,7 @@ def test_log_pathname(test_stage, spec):
     return os.path.join(test_stage, "test-{0}-out.txt".format(TestSuite.test_pkg_id(spec)))
 
 
-class PackageBase(six.with_metaclass(PackageMeta, PackageViewMixin, object)):
+class PackageBase(six.with_metaclass(PackageMeta, WindowsRPathMeta, PackageViewMixin, object)):
     """This is the superclass for all spack packages.
 
     ***The Package class***
@@ -753,6 +780,8 @@ def __init__(self, spec):
         # Set up timing variables
         self._fetch_time = 0.0
 
+        self.win_rpath = fsys.WindowsSimulatedRPath(self)
+
         if self.is_extension:
             pkg_cls = spack.repo.path.get_pkg_class(self.extendee_spec.name)
             pkg_cls(self.extendee_spec)._check_extendable()
@@ -2754,6 +2783,8 @@ def rpath(self):
         deps = self.spec.dependencies(deptype="link")
         rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))
         rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))
+        if is_windows:
+            rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
         return rpaths
 
     @property

lib/spack/spack/test/conftest.py

@@ -27,7 +27,7 @@
 
 import llnl.util.lang
 import llnl.util.tty as tty
-from llnl.util.filesystem import mkdirp, remove_linked_tree, working_dir
+from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir
 
 import spack.binary_distribution
 import spack.caches
@@ -803,7 +803,7 @@ def mock_store(tmpdir_factory, mock_repo_path, mock_configuration_scopes, _store
             with spack.store.use_store(str(store_path)) as store:
                 with spack.repo.use_repositories(mock_repo_path):
                     _populate(store.db)
-        store_path.copy(store_cache, mode=True, stat=True)
+        copy_tree(str(store_path), str(store_cache))
 
     # Make the DB filesystem read-only to ensure we can't modify entries
     store_path.join(".spack-db").chmod(mode=0o555, rec=1)
@@ -844,7 +844,7 @@ def mutable_database(database_mutable_config, _store_dir_and_cache):
     # Restore the initial state by copying the content of the cache back into
     # the store and making the database read-only
     store_path.remove(rec=1)
-    store_cache.copy(store_path, mode=True, stat=True)
+    copy_tree(str(store_cache), str(store_path))
     store_path.join(".spack-db").chmod(mode=0o555, rec=1)
 
 

lib/spack/spack/test/install.py

@@ -5,6 +5,7 @@
 
 import os
 import shutil
+import sys
 
 import pytest
 
@@ -85,12 +86,11 @@ def test_pkg_attributes(install_mockery, mock_fetch, monkeypatch):
     # assert baz_headers.basenames == ['baz.h']
     assert baz_headers.directories == [spec["baz"].home.include]
 
-    if "platform=windows" in spec:
-        lib_suffix = ".lib"
-    elif "platform=darwin" in spec:
+    lib_suffix = ".so"
+    if sys.platform == "win32":
+        lib_suffix = ".dll"
+    elif sys.platform == "darwin":
         lib_suffix = ".dylib"
-    else:
-        lib_suffix = ".so"
 
     foo_libs = spec[foo].libs
     assert foo_libs.basenames == ["libFoo" + lib_suffix]

lib/spack/spack/test/llnl/util/file_list.py

@@ -5,6 +5,7 @@
 
 import fnmatch
 import os.path
+import sys
 
 import pytest
 import six
@@ -19,18 +20,30 @@
 
 import spack.paths
 
+is_windows = sys.platform == "win32"
+
 
 @pytest.fixture()
 def library_list():
     """Returns an instance of LibraryList."""
     # Test all valid extensions: ['.a', '.dylib', '.so']
-    libs = [
-        "/dir1/liblapack.a",
-        "/dir2/libpython3.6.dylib",  # name may contain periods
-        "/dir1/libblas.a",
-        "/dir3/libz.so",
-        "libmpi.so.20.10.1",  # shared object libraries may be versioned
-    ]
+    libs = (
+        [
+            "/dir1/liblapack.a",
+            "/dir2/libpython3.6.dylib",  # name may contain periods
+            "/dir1/libblas.a",
+            "/dir3/libz.so",
+            "libmpi.so.20.10.1",  # shared object libraries may be versioned
+        ]
+        if not is_windows
+        else [
+            "/dir1/liblapack.lib",
+            "/dir2/libpython3.6.dll",
+            "/dir1/libblas.lib",
+            "/dir3/libz.dll",
+            "libmpi.dll.20.10.1",
+        ]
+    )
 
     return LibraryList(libs)
 
@@ -52,6 +65,16 @@ def header_list():
     return h
 
 
+# TODO: Remove below when llnl.util.filesystem.find_libraries becomes spec aware
+plat_static_ext = "lib" if is_windows else "a"
+
+
+plat_shared_ext = "dll" if is_windows else "so"
+
+
+plat_apple_shared_ext = "dll" if is_windows else "dylib"
+
+
 class TestLibraryList(object):
     def test_repr(self, library_list):
         x = eval(repr(library_list))
@@ -62,11 +85,11 @@ def test_joined_and_str(self, library_list):
         s1 = library_list.joined()
         expected = " ".join(
             [
-                "/dir1/liblapack.a",
-                "/dir2/libpython3.6.dylib",
-                "/dir1/libblas.a",
-                "/dir3/libz.so",
-                "libmpi.so.20.10.1",
+                "/dir1/liblapack.%s" % plat_static_ext,
+                "/dir2/libpython3.6.%s" % plat_apple_shared_ext,
+                "/dir1/libblas.%s" % plat_static_ext,
+                "/dir3/libz.%s" % plat_shared_ext,
+                "libmpi.%s.20.10.1" % plat_shared_ext,
             ]
         )
         assert s1 == expected
@@ -77,11 +100,11 @@ def test_joined_and_str(self, library_list):
         s3 = library_list.joined(";")
         expected = ";".join(
             [
-                "/dir1/liblapack.a",
-                "/dir2/libpython3.6.dylib",
-                "/dir1/libblas.a",
-                "/dir3/libz.so",
-                "libmpi.so.20.10.1",
+                "/dir1/liblapack.%s" % plat_static_ext,
+                "/dir2/libpython3.6.%s" % plat_apple_shared_ext,
+                "/dir1/libblas.%s" % plat_static_ext,
+                "/dir3/libz.%s" % plat_shared_ext,
+                "libmpi.%s.20.10.1" % plat_shared_ext,
             ]
         )
         assert s3 == expected
@@ -117,7 +140,7 @@ def test_paths_manipulation(self, library_list):
 
     def test_get_item(self, library_list):
         a = library_list[0]
-        assert a == "/dir1/liblapack.a"
+        assert a == "/dir1/liblapack.%s" % plat_static_ext
 
         b = library_list[:]
         assert type(b) == type(library_list)
@@ -126,9 +149,9 @@ def test_get_item(self, library_list):
 
     def test_add(self, library_list):
         pylist = [
-            "/dir1/liblapack.a",  # removed from the final list
-            "/dir2/libmpi.so",
-            "/dir4/libnew.a",
+            "/dir1/liblapack.%s" % plat_static_ext,  # removed from the final list
+            "/dir2/libmpi.%s" % plat_shared_ext,
+            "/dir4/libnew.%s" % plat_static_ext,
         ]
         another = LibraryList(pylist)
         both = library_list + another

var/spack/repos/builtin.mock/packages/attributes-foo/package.py

@@ -2,6 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import sys
 
 from spack.package import *
 
@@ -14,13 +15,12 @@ class AttributesFoo(BundlePackage):
     provides("baz")
 
     def install(self, spec, prefix):
-        if "platform=windows" in spec:
-            lib_suffix = ".lib"
-        elif "platform=darwin" in spec:
-            lib_suffix = ".dylib"
-        else:
-            lib_suffix = ".so"
 
+        lib_suffix = ".so"
+        if sys.platform == "win32":
+            lib_suffix = ".dll"
+        elif sys.platform == "darwin":
+            lib_suffix = ".dylib"
         mkdirp(prefix.include)
         touch(prefix.include.join("foo.h"))
         mkdirp(prefix.include.bar)