Windows: Symlink support

To provide Windows-compatible functionality, spack code should use
llnl.util.symlink instead of os.symlink. On non-Windows platforms
and on Windows where supported, os.symlink will still be used.

Use junctions when symlinks aren't supported on Windows (#22583)

Support islink for junctions (#24182)

Windows: Update llnl/util/filesystem

* Use '/' as path separator on Windows.
* Recognizing that Windows paths start with '<Letter>:/' instead of '/'

Co-authored-by: lou.lawrence@kitware.com <lou.lawrence@kitware.com>
Co-authored-by: John Parent <john.parent@kitware.com>

lib/spack/external/macholib/util.py

@@ -6,6 +6,8 @@
 
 from macholib import mach_o
 
+from llnl.util.symlink import symlink
+
 MAGIC = [
     struct.pack("!L", getattr(mach_o, "MH_" + _))
     for _ in ["MAGIC", "CIGAM", "MAGIC_64", "CIGAM_64"]
@@ -140,7 +142,7 @@ def mergetree(src, dst, condition=None, copyfn=mergecopy, srcbase=None):
         try:
             if os.path.islink(srcname):
                 realsrc = os.readlink(srcname)
-                os.symlink(realsrc, dstname)
+                symlink(realsrc, dstname)
             elif os.path.isdir(srcname):
                 mergetree(
                     srcname,

lib/spack/external/pytest-fallback/py/_path/local.py

@@ -12,6 +12,8 @@
 
 from os.path import abspath, normpath, isabs, exists, isdir, isfile, islink, dirname
 
+from llnl.util.symlink import symlink
+
 if sys.version_info > (3,0):
     def map_as_list(func, iter):
         return list(map(func, iter))
@@ -79,7 +81,7 @@ def mklinkto(self, oldname):
     def mksymlinkto(self, value, absolute=1):
         """ create a symbolic link with the given value (pointing to another name). """
         if absolute:
-            py.error.checked_call(os.symlink, str(value), self.strpath)
+            py.error.checked_call(symlink, str(value), self.strpath)
         else:
             base = self.common(value)
             # with posix local paths '/' is always a common base
@@ -87,7 +89,7 @@ def mksymlinkto(self, value, absolute=1):
             reldest = self.relto(base)
             n = reldest.count(self.sep)
             target = self.sep.join(('..', )*n + (relsource, ))
-            py.error.checked_call(os.symlink, target, self.strpath)
+            py.error.checked_call(symlink, target, self.strpath)
 
 def getuserid(user):
     import pwd
@@ -892,7 +894,7 @@ def try_remove_lockfile():
         except OSError:
             pass
         try:
-            os.symlink(src, dest)
+            symlink(src, dest)
         except (OSError, AttributeError, NotImplementedError):
             pass
 

lib/spack/llnl/util/filesystem.py

@@ -24,6 +24,7 @@
 from llnl.util import tty
 from llnl.util.compat import Sequence
 from llnl.util.lang import dedupe, memoized
+from llnl.util.symlink import symlink
 
 from spack.util.executable import Executable
 
@@ -508,7 +509,7 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
                                       .format(target, new_target))
                             target = new_target
 
-                    os.symlink(target, d)
+                    symlink(target, d)
                 elif os.path.isdir(link_target):
                     mkdirp(d)
                 else:
@@ -806,10 +807,10 @@ def touchp(path):
 
 def force_symlink(src, dest):
     try:
-        os.symlink(src, dest)
+        symlink(src, dest)
     except OSError:
         os.remove(dest)
-        os.symlink(src, dest)
+        symlink(src, dest)
 
 
 def join_path(prefix, *args):

lib/spack/llnl/util/link_tree.py

@@ -13,6 +13,7 @@
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp, touch, traverse_tree
+from llnl.util.symlink import islink, symlink
 
 __all__ = ['LinkTree']
 
@@ -20,7 +21,7 @@
 
 
 def remove_link(src, dest):
-    if not os.path.islink(dest):
+    if not islink(dest):
         raise ValueError("%s is not a link tree!" % dest)
     # remove if dest is a hardlink/symlink to src; this will only
     # be false if two packages are merged into a prefix and have a
@@ -113,7 +114,7 @@ def unmerge_directories(self, dest_root, ignore):
                     os.remove(marker)
 
     def merge(self, dest_root, ignore_conflicts=False, ignore=None,
-              link=os.symlink, relative=False):
+              link=symlink, relative=False):
         """Link all files in src into dest, creating directories
            if necessary.
 
@@ -125,7 +126,7 @@ def merge(self, dest_root, ignore_conflicts=False, ignore=None,
         ignore (callable): callable that returns True if a file is to be
             ignored in the merge (by default ignore nothing)
 
-        link (callable): function to create links with (defaults to os.symlink)
+        link (callable): function to create links with (defaults to llnl.util.symlink)
 
         relative (bool): create all symlinks relative to the target
             (default False)

lib/spack/llnl/util/symlink.py

@@ -0,0 +1,139 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import errno
+import os
+import shutil
+import tempfile
+from os.path import exists, join
+from sys import platform as _platform
+
+is_windows = _platform == 'win32'
+
+__win32_can_symlink__ = None
+
+
+def symlink(real_path, link_path):
+    """
+    Create a symbolic link.
+
+    On Windows, use junctions if os.symlink fails.
+    """
+    if not is_windows or _win32_can_symlink():
+        os.symlink(real_path, link_path)
+    else:
+        try:
+            # Try to use junctions
+            _win32_junction(real_path, link_path)
+        except OSError:
+            # If all else fails, fall back to copying files
+            shutil.copyfile(real_path, link_path)
+
+
+def islink(path):
+    return os.path.islink(path) or _win32_is_junction(path)
+
+
+# '_win32' functions based on
+# https://github.com/Erotemic/ubelt/blob/master/ubelt/util_links.py
+def _win32_junction(path, link):
+    # junctions require absolute paths
+    if not os.path.isabs(link):
+        link = os.path.abspath(link)
+
+    # os.symlink will fail if link exists, emulate the behavior here
+    if exists(link):
+        raise OSError(errno.EEXIST, 'File  exists: %s -> %s' % (link, path))
+
+    if not os.path.isabs(path):
+        parent = os.path.join(link, os.pardir)
+        path = os.path.join(parent, path)
+        path = os.path.abspath(path)
+
+    if os.path.isdir(path):
+        # try using a junction
+        command = 'mklink /J "%s" "%s"' % (link, path)
+    else:
+        # try using a hard link
+        command = 'mklink /H "%s" "%s"' % (link, path)
+
+    _cmd(command)
+
+
+def _win32_can_symlink():
+    global __win32_can_symlink__
+    if __win32_can_symlink__ is not None:
+        return __win32_can_symlink__
+
+    tempdir = tempfile.mkdtemp()
+
+    dpath = join(tempdir, 'dpath')
+    fpath = join(tempdir, 'fpath.txt')
+
+    dlink = join(tempdir, 'dlink')
+    flink = join(tempdir, 'flink.txt')
+
+    import llnl.util.filesystem as fs
+    fs.touchp(fpath)
+
+    try:
+        os.symlink(dpath, dlink)
+        can_symlink_directories = os.path.islink(dlink)
+    except OSError:
+        can_symlink_directories = False
+
+    try:
+        os.symlink(fpath, flink)
+        can_symlink_files = os.path.islink(flink)
+    except OSError:
+        can_symlink_files = False
+
+    # Cleanup the test directory
+    shutil.rmtree(tempdir)
+
+    __win32_can_symlink__ = can_symlink_directories and can_symlink_files
+
+    return __win32_can_symlink__
+
+
+def _win32_is_junction(path):
+    """
+    Determines if a path is a win32 junction
+    """
+    if os.path.islink(path):
+        return False
+
+    if is_windows:
+        import ctypes.wintypes
+
+        GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW
+        GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,)
+        GetFileAttributes.restype = ctypes.wintypes.DWORD
+
+        INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
+        FILE_ATTRIBUTE_REPARSE_POINT = 0x400
+
+        res = GetFileAttributes(path)
+        return res != INVALID_FILE_ATTRIBUTES and \
+            bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
+
+    return False
+
+
+# Based on https://github.com/Erotemic/ubelt/blob/master/ubelt/util_cmd.py
+def _cmd(command):
+    import subprocess
+
+    # Create a new process to execute the command
+    def make_proc():
+        # delay the creation of the process until we validate all args
+        proc = subprocess.Popen(command, stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE, shell=True,
+                                universal_newlines=True, cwd=None, env=None)
+        return proc
+
+    proc = make_proc()
+    (out, err) = proc.communicate()
+    if proc.wait() != 0:
+        raise OSError(str(err))

lib/spack/spack/build_environment.py

@@ -46,6 +46,7 @@
 import llnl.util.tty as tty
 from llnl.util.filesystem import install, install_tree, mkdirp
 from llnl.util.lang import dedupe
+from llnl.util.symlink import symlink
 from llnl.util.tty.color import cescape, colorize
 from llnl.util.tty.log import MultiProcessFd
 
@@ -545,7 +546,7 @@ def _set_variables_for_single_module(pkg, module):
     m.makedirs = os.makedirs
     m.remove = os.remove
     m.removedirs = os.removedirs
-    m.symlink = os.symlink
+    m.symlink = symlink
 
     m.mkdirp = mkdirp
     m.install = install
@@ -668,10 +669,10 @@ def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None,
     shared_lib_link = os.path.basename(shared_lib)
 
     if version or compat_version:
-        os.symlink(shared_lib_link, shared_lib_base)
+        symlink(shared_lib_link, shared_lib_base)
 
     if compat_version and compat_version != version:
-        os.symlink(shared_lib_link, '{0}.{1}'.format(shared_lib_base,
+        symlink(shared_lib_link, '{0}.{1}'.format(shared_lib_base,
                                                   compat_version))
 
     return compiler(*compiler_args, output=compiler_output)

lib/spack/spack/caches.py

@@ -8,6 +8,7 @@
 
 import llnl.util.lang
 from llnl.util.filesystem import mkdirp
+from llnl.util.symlink import symlink
 
 import spack.config
 import spack.error
@@ -85,7 +86,7 @@ def symlink(self, mirror_ref):
                 # to https://github.com/spack/spack/pull/13908)
                 os.unlink(cosmetic_path)
             mkdirp(os.path.dirname(cosmetic_path))
-            os.symlink(relative_dst, cosmetic_path)
+            symlink(relative_dst, cosmetic_path)
 
 
 #: Spack's local cache for downloaded source archives

lib/spack/spack/cmd/deprecate.py

@@ -19,6 +19,7 @@
 import os
 
 import llnl.util.tty as tty
+from llnl.util.symlink import symlink
 
 import spack.cmd
 import spack.cmd.common.arguments as arguments
@@ -123,7 +124,7 @@ def deprecate(parser, args):
         if not answer:
             tty.die('Will not deprecate any packages.')
 
-    link_fn = os.link if args.link_type == 'hard' else os.symlink
+    link_fn = os.link if args.link_type == 'hard' else symlink
 
     for dcate, dcator in zip(all_deprecate, all_deprecators):
         dcate.package.do_deprecate(dcator, link_fn)

lib/spack/spack/cmd/modules/lmod.py

@@ -4,6 +4,10 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 import functools
+import os
+
+import llnl.util.filesystem
+from llnl.util.symlink import symlink
 
 import spack.cmd.common.arguments
 import spack.cmd.modules
@@ -56,3 +60,11 @@ def setdefault(module_type, specs, args):
     with spack.config.override(scope):
         writer = spack.modules.module_types['lmod'](spec, args.module_set_name)
         writer.update_module_defaults()
+
+
+    module_folder = os.path.dirname(writer.layout.filename)
+    module_basename = os.path.basename(writer.layout.filename)
+    with llnl.util.filesystem.working_dir(module_folder):
+        if os.path.exists('default') and os.path.islink('default'):
+            os.remove('default')
+        symlink(module_basename, 'default')

lib/spack/spack/compilers/apple_clang.py

@@ -8,6 +8,7 @@
 
 import llnl.util.lang
 import llnl.util.tty as tty
+from llnl.util.symlink import symlink
 
 import spack.compiler
 import spack.compilers.clang
@@ -162,10 +163,10 @@ def setup_custom_environment(self, pkg, env):
                 for fname in os.listdir(dev_dir):
                     if fname in bins:
                         os.unlink(os.path.join(dev_dir, fname))
-                        os.symlink(
+                        symlink(
                             os.path.join(spack.paths.build_env_path, 'cc'),
                             os.path.join(dev_dir, fname))
 
-            os.symlink(developer_root, xcode_link)
+            symlink(developer_root, xcode_link)
 
         env.set('DEVELOPER_DIR', xcode_link)

lib/spack/spack/config.py

@@ -293,6 +293,7 @@ def _write_section(self, section):
                 syaml.dump_config(data_to_write, stream=f,
                                   default_flow_style=False)
             rename(tmp, self.path)
+
         except (yaml.YAMLError, IOError) as e:
             raise ConfigFileError(
                 "Error writing to config file: '%s'" % str(e))

lib/spack/spack/environment/environment.py

@@ -17,6 +17,7 @@
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
 from llnl.util.lang import dedupe
+from llnl.util.symlink import islink, symlink
 
 import spack.bootstrap
 import spack.compilers
@@ -1461,7 +1462,7 @@ def _install_log_links(self, spec):
                     log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
                 if os.path.lexists(build_log_link):
                     os.remove(build_log_link)
-                os.symlink(spec.package.build_log_path, build_log_link)
+                symlink(spec.package.build_log_path, build_log_link)
 
     def uninstalled_specs(self):
         """Return a list of all uninstalled (and non-dev) specs."""

lib/spack/spack/fetch_strategy.py

@@ -43,6 +43,7 @@
     temp_rename,
     working_dir,
 )
+from llnl.util.symlink import symlink
 
 import spack.config
 import spack.error
@@ -640,7 +641,7 @@ def fetch(self):
             os.remove(filename)
 
         # Symlink to local cached archive.
-        os.symlink(path, filename)
+        symlink(path, filename)
 
         # Remove link if checksum fails, or subsequent fetchers
         # will assume they don't need to download.

lib/spack/spack/filesystem_view.py

@@ -15,6 +15,7 @@
 from llnl.util.filesystem import mkdirp, remove_dead_links, remove_empty_directories
 from llnl.util.lang import index_by, match_predicate
 from llnl.util.link_tree import LinkTree, MergeConflictError
+from llnl.util.symlink import symlink
 from llnl.util.tty.color import colorize
 
 import spack.config
@@ -39,7 +40,7 @@
 def view_symlink(src, dst, **kwargs):
     # keyword arguments are irrelevant
     # here to fit required call signature
-    os.symlink(src, dst)
+    symlink(src, dst)
 
 
 def view_hardlink(src, dst, **kwargs):
@@ -141,7 +142,7 @@ def __init__(self, root, layout, **kwargs):
             Initialize a filesystem view under the given `root` directory with
             corresponding directory `layout`.
 
-            Files are linked by method `link` (os.symlink by default).
+            Files are linked by method `link` (llnl.util.symlink by default).
         """
         self._root = root
         self.layout = layout

lib/spack/spack/hooks/licensing.py

@@ -7,6 +7,7 @@
 
 import llnl.util.tty as tty
 from llnl.util.filesystem import mkdirp
+from llnl.util.symlink import symlink
 
 from spack.util.editor import editor
 from spack.util.executable import Executable, which
@@ -179,6 +180,6 @@ def symlink_license(pkg):
             os.remove(link_name)
 
         if os.path.exists(target):
-            os.symlink(target, link_name)
+            symlink(target, link_name)
             tty.msg("Added local symlink %s to global license file" %
                     link_name)

lib/spack/spack/relocate.py

@@ -13,6 +13,7 @@
 
 import llnl.util.lang
 import llnl.util.tty as tty
+from llnl.util.symlink import symlink
 
 import spack.bootstrap
 import spack.platforms
@@ -683,7 +684,7 @@ def make_link_relative(new_links, orig_links):
         target = os.readlink(orig_link)
         relative_target = os.path.relpath(target, os.path.dirname(orig_link))
         os.unlink(new_link)
-        os.symlink(relative_target, new_link)
+        symlink(relative_target, new_link)
 
 
 def make_macho_binaries_relative(cur_path_names, orig_path_names,
@@ -764,7 +765,7 @@ def relocate_links(links, orig_layout_root,
                 orig_install_prefix, new_install_prefix, link_target
             )
             os.unlink(abs_link)
-            os.symlink(link_target, abs_link)
+            symlink(link_target, abs_link)
 
         # If the link is absolute and has not been relocated then
         # warn the user about that

lib/spack/spack/test/llnl/util/filesystem.py

@@ -12,6 +12,7 @@
 import pytest
 
 import llnl.util.filesystem as fs
+from llnl.util.symlink import symlink
 
 import spack.paths
 
@@ -36,9 +37,9 @@ def stage(tmpdir_factory):
         fs.touchp('source/g/i/j/10')
 
         # Create symlinks
-        os.symlink(os.path.abspath('source/1'), 'source/2')
+        symlink(os.path.abspath('source/1'), 'source/2')
-        os.symlink('b/2', 'source/a/b2')
+        symlink('b/2', 'source/a/b2')
-        os.symlink('a/b', 'source/f')
+        symlink('a/b', 'source/f')
 
         # Create destination directory
         fs.mkdirp('dest')
@@ -174,12 +175,18 @@ def test_symlinks_true(self, stage):
             fs.copy_tree('source', 'dest', symlinks=True)
 
             assert os.path.exists('dest/2')
+            if sys.platform != "win32":
+                # TODO: islink will return false for junctions
                 assert os.path.islink('dest/2')
 
             assert os.path.exists('dest/a/b2')
+            if sys.platform != "win32":
+                # TODO: Not supported for junctions ?
                 with fs.working_dir('dest/a'):
                     assert os.path.exists(os.readlink('b2'))
 
+            if sys.platform != "win32":
+                # TODO: Not supported on Windows ?
                 assert (os.path.realpath('dest/f/2') ==
                         os.path.abspath('dest/a/b/2'))
                 assert os.path.realpath('dest/2') == os.path.abspath('dest/1')
@@ -201,6 +208,7 @@ def test_symlinks_false(self, stage):
             fs.copy_tree('source', 'dest', symlinks=False)
 
             assert os.path.exists('dest/2')
+            if sys.platform != "win32":
                 assert not os.path.islink('dest/2')
 
     def test_glob_src(self, stage):
@@ -258,6 +266,7 @@ def test_symlinks_true(self, stage):
             fs.install_tree('source', 'dest', symlinks=True)
 
             assert os.path.exists('dest/2')
+            if sys.platform != "win32":
                 assert os.path.islink('dest/2')
             check_added_exe_permissions('source/2', 'dest/2')
 
@@ -268,6 +277,7 @@ def test_symlinks_false(self, stage):
             fs.install_tree('source', 'dest', symlinks=False)
 
             assert os.path.exists('dest/2')
+            if sys.platform != "win32":
                 assert not os.path.islink('dest/2')
             check_added_exe_permissions('source/2', 'dest/2')
 

lib/spack/spack/test/llnl/util/link_tree.py

@@ -9,6 +9,7 @@
 
 from llnl.util.filesystem import mkdirp, touchp, working_dir
 from llnl.util.link_tree import LinkTree
+from llnl.util.symlink import islink
 
 from spack.stage import Stage
 
@@ -42,7 +43,7 @@ def link_tree(stage):
 
 def check_file_link(filename, expected_target):
     assert os.path.isfile(filename)
-    assert os.path.islink(filename)
+    assert islink(filename)
     assert (os.path.abspath(os.path.realpath(filename)) ==
             os.path.abspath(expected_target))
 

lib/spack/spack/test/packaging.py

@@ -16,6 +16,7 @@
 import pytest
 
 from llnl.util.filesystem import mkdirp
+from llnl.util.symlink import symlink
 
 import spack.binary_distribution as bindist
 import spack.cmd.buildcache as buildcache
@@ -79,7 +80,7 @@ def test_buildcache(mock_archive, tmpdir):
 
     # Create an absolute symlink
     linkname = os.path.join(spec.prefix, "link_to_dummy.txt")
-    os.symlink(filename, linkname)
+    symlink(filename, linkname)
 
     # Create the build cache  and
     # put it directly into the mirror
@@ -232,8 +233,8 @@ def test_relocate_links(tmpdir):
         with open(new_binname, 'w') as f:
             f.write('\n')
         os.utime(new_binname, None)
-        os.symlink(old_binname, new_linkname)
+        symlink(old_binname, new_linkname)
-        os.symlink('/usr/lib/libc.so', new_linkname2)
+        symlink('/usr/lib/libc.so', new_linkname2)
         relocate_links(filenames, old_layout_root,
                        old_install_prefix, new_install_prefix)
         assert os.readlink(new_linkname) == new_binname

lib/spack/spack/test/verification.py

@@ -8,6 +8,7 @@
 import shutil
 
 import llnl.util.filesystem as fs
+from llnl.util.symlink import symlink
 
 import spack.spec
 import spack.store
@@ -21,7 +22,7 @@ def test_link_manifest_entry(tmpdir):
     file = str(tmpdir.join('file'))
     open(file, 'a').close()
     link = str(tmpdir.join('link'))
-    os.symlink(file, link)
+    symlink(file, link)
 
     data = spack.verify.create_manifest_entry(link)
     assert data['type'] == 'link'
@@ -43,7 +44,7 @@ def test_link_manifest_entry(tmpdir):
     file2 = str(tmpdir.join('file2'))
     open(file2, 'a').close()
     os.remove(link)
-    os.symlink(file2, link)
+    symlink(file2, link)
 
     results = spack.verify.check_entry(link, data)
     assert results.has_errors()
@@ -157,7 +158,7 @@ def test_check_prefix_manifest(tmpdir):
         f.write("I'm a little file short and stout")
 
     link = os.path.join(bin_dir, 'run')
-    os.symlink(file, link)
+    symlink(file, link)
 
     spack.verify.write_manifest(spec)
     results = spack.verify.check_spec_manifest(spec)