hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Windows: Symlink support

Browse source 
To provide Windows-compatible functionality, spack code should use
llnl.util.symlink instead of os.symlink. On non-Windows platforms
and on Windows where supported, os.symlink will still be used.

Use junctions when symlinks aren't supported on Windows (#22583)

Support islink for junctions (#24182)

Windows: Update llnl/util/filesystem

* Use '/' as path separator on Windows.
* Recognizing that Windows paths start with '<Letter>:/' instead of '/'

Co-authored-by: lou.lawrence@kitware.com <lou.lawrence@kitware.com>
Co-authored-by: John Parent <john.parent@kitware.com>
This commit is contained in:
Betsy McPhail 2021-10-22 12:16:11 -04:00 committed by Peter Scheibel
parent a7de2fa380
commit fb0e91c534
20 changed files with 224 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
lib/spack/external/macholib/util.py vendored
View file

@ -6,6 +6,8 @@
from macholib import mach_o
from llnl.util.symlink import symlink
MAGIC = [
struct.pack("!L", getattr(mach_o, "MH_" + _))
for _ in ["MAGIC", "CIGAM", "MAGIC_64", "CIGAM_64"]
@ -140,7 +142,7 @@ def mergetree(src, dst, condition=None, copyfn=mergecopy, srcbase=None):
try:
if os.path.islink(srcname):
realsrc = os.readlink(srcname)
os.symlink(realsrc, dstname)
symlink(realsrc, dstname)
elif os.path.isdir(srcname):
mergetree(
srcname,

8
lib/spack/external/pytest-fallback/py/_path/local.py vendored
View file

@ -12,6 +12,8 @@
from os.path import abspath, normpath, isabs, exists, isdir, isfile, islink, dirname
from llnl.util.symlink import symlink
if sys.version_info > (3,0):
def map_as_list(func, iter):
return list(map(func, iter))
@ -79,7 +81,7 @@ def mklinkto(self, oldname):
def mksymlinkto(self, value, absolute=1):
""" create a symbolic link with the given value (pointing to another name). """
if absolute:
py.error.checked_call(os.symlink, str(value), self.strpath)
py.error.checked_call(symlink, str(value), self.strpath)
else:
base = self.common(value)
# with posix local paths '/' is always a common base
@ -87,7 +89,7 @@ def mksymlinkto(self, value, absolute=1):
reldest = self.relto(base)
n = reldest.count(self.sep)
target = self.sep.join(('..', )*n + (relsource, ))
py.error.checked_call(os.symlink, target, self.strpath)
py.error.checked_call(symlink, target, self.strpath)
def getuserid(user):
import pwd
@ -892,7 +894,7 @@ def try_remove_lockfile():
except OSError:
pass
try:
os.symlink(src, dest)
symlink(src, dest)
except (OSError, AttributeError, NotImplementedError):
pass

7
lib/spack/llnl/util/filesystem.py
View file

@ -24,6 +24,7 @@
from llnl.util import tty
from llnl.util.compat import Sequence
from llnl.util.lang import dedupe, memoized
from llnl.util.symlink import symlink
from spack.util.executable import Executable
@ -508,7 +509,7 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
.format(target, new_target))
target = new_target
os.symlink(target, d)
symlink(target, d)
elif os.path.isdir(link_target):
mkdirp(d)
else:
@ -806,10 +807,10 @@ def touchp(path):
def force_symlink(src, dest):
try:
os.symlink(src, dest)
symlink(src, dest)
except OSError:
os.remove(dest)
os.symlink(src, dest)
symlink(src, dest)
def join_path(prefix, *args):

7
lib/spack/llnl/util/link_tree.py
View file

@ -13,6 +13,7 @@
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp, touch, traverse_tree
from llnl.util.symlink import islink, symlink
__all__ = ['LinkTree']
@ -20,7 +21,7 @@
def remove_link(src, dest):
if not os.path.islink(dest):
if not islink(dest):
raise ValueError("%s is not a link tree!" % dest)
# remove if dest is a hardlink/symlink to src; this will only
# be false if two packages are merged into a prefix and have a
@ -113,7 +114,7 @@ def unmerge_directories(self, dest_root, ignore):
os.remove(marker)
def merge(self, dest_root, ignore_conflicts=False, ignore=None,
link=os.symlink, relative=False):
link=symlink, relative=False):
"""Link all files in src into dest, creating directories
if necessary.
@ -125,7 +126,7 @@ def merge(self, dest_root, ignore_conflicts=False, ignore=None,
ignore (callable): callable that returns True if a file is to be
ignored in the merge (by default ignore nothing)
link (callable): function to create links with (defaults to os.symlink)
link (callable): function to create links with (defaults to llnl.util.symlink)
relative (bool): create all symlinks relative to the target
(default False)

139
lib/spack/llnl/util/symlink.py Normal file
View file

@ -0,0 +1,139 @@
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import errno
import os
import shutil
import tempfile
from os.path import exists, join
from sys import platform as _platform
is_windows = _platform == 'win32'
__win32_can_symlink__ = None
def symlink(real_path, link_path):
"""
Create a symbolic link.
On Windows, use junctions if os.symlink fails.
"""
if not is_windows or _win32_can_symlink():
os.symlink(real_path, link_path)
else:
try:
# Try to use junctions
_win32_junction(real_path, link_path)
except OSError:
# If all else fails, fall back to copying files
shutil.copyfile(real_path, link_path)
def islink(path):
return os.path.islink(path) or _win32_is_junction(path)
# '_win32' functions based on
# https://github.com/Erotemic/ubelt/blob/master/ubelt/util_links.py
def _win32_junction(path, link):
# junctions require absolute paths
if not os.path.isabs(link):
link = os.path.abspath(link)
# os.symlink will fail if link exists, emulate the behavior here
if exists(link):
raise OSError(errno.EEXIST, 'File exists: %s -> %s' % (link, path))
if not os.path.isabs(path):
parent = os.path.join(link, os.pardir)
path = os.path.join(parent, path)
path = os.path.abspath(path)
if os.path.isdir(path):
# try using a junction
command = 'mklink /J "%s" "%s"' % (link, path)
else:
# try using a hard link
command = 'mklink /H "%s" "%s"' % (link, path)
_cmd(command)
def _win32_can_symlink():
global __win32_can_symlink__
if __win32_can_symlink__ is not None:
return __win32_can_symlink__
tempdir = tempfile.mkdtemp()
dpath = join(tempdir, 'dpath')
fpath = join(tempdir, 'fpath.txt')
dlink = join(tempdir, 'dlink')
flink = join(tempdir, 'flink.txt')
import llnl.util.filesystem as fs
fs.touchp(fpath)
try:
os.symlink(dpath, dlink)
can_symlink_directories = os.path.islink(dlink)
except OSError:
can_symlink_directories = False
try:
os.symlink(fpath, flink)
can_symlink_files = os.path.islink(flink)
except OSError:
can_symlink_files = False
# Cleanup the test directory
shutil.rmtree(tempdir)
__win32_can_symlink__ = can_symlink_directories and can_symlink_files
return __win32_can_symlink__
def _win32_is_junction(path):
"""
Determines if a path is a win32 junction
"""
if os.path.islink(path):
return False
if is_windows:
import ctypes.wintypes
GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW
GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,)
GetFileAttributes.restype = ctypes.wintypes.DWORD
INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
res = GetFileAttributes(path)
return res != INVALID_FILE_ATTRIBUTES and \
bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
return False
# Based on https://github.com/Erotemic/ubelt/blob/master/ubelt/util_cmd.py
def _cmd(command):
import subprocess
# Create a new process to execute the command
def make_proc():
# delay the creation of the process until we validate all args
proc = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, shell=True,
universal_newlines=True, cwd=None, env=None)
return proc
proc = make_proc()
(out, err) = proc.communicate()
if proc.wait() != 0:
raise OSError(str(err))

7
lib/spack/spack/build_environment.py
View file

@ -46,6 +46,7 @@
import llnl.util.tty as tty
from llnl.util.filesystem import install, install_tree, mkdirp
from llnl.util.lang import dedupe
from llnl.util.symlink import symlink
from llnl.util.tty.color import cescape, colorize
from llnl.util.tty.log import MultiProcessFd
@ -545,7 +546,7 @@ def _set_variables_for_single_module(pkg, module):
m.makedirs = os.makedirs
m.remove = os.remove
m.removedirs = os.removedirs
m.symlink = os.symlink
m.symlink = symlink
m.mkdirp = mkdirp
m.install = install
@ -668,10 +669,10 @@ def _static_to_shared_library(arch, compiler, static_lib, shared_lib=None,
shared_lib_link = os.path.basename(shared_lib)
if version or compat_version:
os.symlink(shared_lib_link, shared_lib_base)
symlink(shared_lib_link, shared_lib_base)
if compat_version and compat_version != version:
os.symlink(shared_lib_link, '{0}.{1}'.format(shared_lib_base,
symlink(shared_lib_link, '{0}.{1}'.format(shared_lib_base,
compat_version))
return compiler(*compiler_args, output=compiler_output)

3
lib/spack/spack/caches.py
View file

@ -8,6 +8,7 @@
import llnl.util.lang
from llnl.util.filesystem import mkdirp
from llnl.util.symlink import symlink
import spack.config
import spack.error
@ -85,7 +86,7 @@ def symlink(self, mirror_ref):
# to https://github.com/spack/spack/pull/13908)
os.unlink(cosmetic_path)
mkdirp(os.path.dirname(cosmetic_path))
os.symlink(relative_dst, cosmetic_path)
symlink(relative_dst, cosmetic_path)
#: Spack's local cache for downloaded source archives

3
lib/spack/spack/cmd/deprecate.py
View file

@ -19,6 +19,7 @@
import os
import llnl.util.tty as tty
from llnl.util.symlink import symlink
import spack.cmd
import spack.cmd.common.arguments as arguments
@ -123,7 +124,7 @@ def deprecate(parser, args):
if not answer:
tty.die('Will not deprecate any packages.')
link_fn = os.link if args.link_type == 'hard' else os.symlink
link_fn = os.link if args.link_type == 'hard' else symlink
for dcate, dcator in zip(all_deprecate, all_deprecators):
dcate.package.do_deprecate(dcator, link_fn)

12
lib/spack/spack/cmd/modules/lmod.py
View file

@ -4,6 +4,10 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import functools
import os
import llnl.util.filesystem
from llnl.util.symlink import symlink
import spack.cmd.common.arguments
import spack.cmd.modules
@ -56,3 +60,11 @@ def setdefault(module_type, specs, args):
with spack.config.override(scope):
writer = spack.modules.module_types['lmod'](spec, args.module_set_name)
writer.update_module_defaults()
module_folder = os.path.dirname(writer.layout.filename)
module_basename = os.path.basename(writer.layout.filename)
with llnl.util.filesystem.working_dir(module_folder):
if os.path.exists('default') and os.path.islink('default'):
os.remove('default')
symlink(module_basename, 'default')

5
lib/spack/spack/compilers/apple_clang.py
View file

@ -8,6 +8,7 @@
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.symlink import symlink
import spack.compiler
import spack.compilers.clang
@ -162,10 +163,10 @@ def setup_custom_environment(self, pkg, env):
for fname in os.listdir(dev_dir):
if fname in bins:
os.unlink(os.path.join(dev_dir, fname))
os.symlink(
symlink(
os.path.join(spack.paths.build_env_path, 'cc'),
os.path.join(dev_dir, fname))
os.symlink(developer_root, xcode_link)
symlink(developer_root, xcode_link)
env.set('DEVELOPER_DIR', xcode_link)

1
lib/spack/spack/config.py
View file

@ -293,6 +293,7 @@ def _write_section(self, section):
syaml.dump_config(data_to_write, stream=f,
default_flow_style=False)
rename(tmp, self.path)
except (yaml.YAMLError, IOError) as e:
raise ConfigFileError(
"Error writing to config file: '%s'" % str(e))

3
lib/spack/spack/environment/environment.py
View file

@ -17,6 +17,7 @@
import llnl.util.filesystem as fs
import llnl.util.tty as tty
from llnl.util.lang import dedupe
from llnl.util.symlink import islink, symlink
import spack.bootstrap
import spack.compilers
@ -1461,7 +1462,7 @@ def _install_log_links(self, spec):
log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
if os.path.lexists(build_log_link):
os.remove(build_log_link)
os.symlink(spec.package.build_log_path, build_log_link)
symlink(spec.package.build_log_path, build_log_link)
def uninstalled_specs(self):
"""Return a list of all uninstalled (and non-dev) specs."""

3
lib/spack/spack/fetch_strategy.py
View file

@ -43,6 +43,7 @@
temp_rename,
working_dir,
)
from llnl.util.symlink import symlink
import spack.config
import spack.error
@ -640,7 +641,7 @@ def fetch(self):
os.remove(filename)
# Symlink to local cached archive.
os.symlink(path, filename)
symlink(path, filename)
# Remove link if checksum fails, or subsequent fetchers
# will assume they don't need to download.

5
lib/spack/spack/filesystem_view.py
View file

@ -15,6 +15,7 @@
from llnl.util.filesystem import mkdirp, remove_dead_links, remove_empty_directories
from llnl.util.lang import index_by, match_predicate
from llnl.util.link_tree import LinkTree, MergeConflictError
from llnl.util.symlink import symlink
from llnl.util.tty.color import colorize
import spack.config
@ -39,7 +40,7 @@
def view_symlink(src, dst, **kwargs):
# keyword arguments are irrelevant
# here to fit required call signature
os.symlink(src, dst)
symlink(src, dst)
def view_hardlink(src, dst, **kwargs):
@ -141,7 +142,7 @@ def __init__(self, root, layout, **kwargs):
Initialize a filesystem view under the given `root` directory with
corresponding directory `layout`.
Files are linked by method `link` (os.symlink by default).
Files are linked by method `link` (llnl.util.symlink by default).
"""
self._root = root
self.layout = layout

3
lib/spack/spack/hooks/licensing.py
View file

@ -7,6 +7,7 @@
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp
from llnl.util.symlink import symlink
from spack.util.editor import editor
from spack.util.executable import Executable, which
@ -179,6 +180,6 @@ def symlink_license(pkg):
os.remove(link_name)
if os.path.exists(target):
os.symlink(target, link_name)
symlink(target, link_name)
tty.msg("Added local symlink %s to global license file" %
link_name)

5
lib/spack/spack/relocate.py
View file

@ -13,6 +13,7 @@
import llnl.util.lang
import llnl.util.tty as tty
from llnl.util.symlink import symlink
import spack.bootstrap
import spack.platforms
@ -683,7 +684,7 @@ def make_link_relative(new_links, orig_links):
target = os.readlink(orig_link)
relative_target = os.path.relpath(target, os.path.dirname(orig_link))
os.unlink(new_link)
os.symlink(relative_target, new_link)
symlink(relative_target, new_link)
def make_macho_binaries_relative(cur_path_names, orig_path_names,
@ -764,7 +765,7 @@ def relocate_links(links, orig_layout_root,
orig_install_prefix, new_install_prefix, link_target
)
os.unlink(abs_link)
os.symlink(link_target, abs_link)
symlink(link_target, abs_link)
# If the link is absolute and has not been relocated then
# warn the user about that

16
lib/spack/spack/test/llnl/util/filesystem.py
View file

@ -12,6 +12,7 @@
import pytest
import llnl.util.filesystem as fs
from llnl.util.symlink import symlink
import spack.paths
@ -36,9 +37,9 @@ def stage(tmpdir_factory):
fs.touchp('source/g/i/j/10')
# Create symlinks
os.symlink(os.path.abspath('source/1'), 'source/2')
os.symlink('b/2', 'source/a/b2')
os.symlink('a/b', 'source/f')
symlink(os.path.abspath('source/1'), 'source/2')
symlink('b/2', 'source/a/b2')
symlink('a/b', 'source/f')
# Create destination directory
fs.mkdirp('dest')
@ -174,12 +175,18 @@ def test_symlinks_true(self, stage):
fs.copy_tree('source', 'dest', symlinks=True)
assert os.path.exists('dest/2')
if sys.platform != "win32":
# TODO: islink will return false for junctions
assert os.path.islink('dest/2')
assert os.path.exists('dest/a/b2')
if sys.platform != "win32":
# TODO: Not supported for junctions ?
with fs.working_dir('dest/a'):
assert os.path.exists(os.readlink('b2'))
if sys.platform != "win32":
# TODO: Not supported on Windows ?
assert (os.path.realpath('dest/f/2') ==
os.path.abspath('dest/a/b/2'))
assert os.path.realpath('dest/2') == os.path.abspath('dest/1')
@ -201,6 +208,7 @@ def test_symlinks_false(self, stage):
fs.copy_tree('source', 'dest', symlinks=False)
assert os.path.exists('dest/2')
if sys.platform != "win32":
assert not os.path.islink('dest/2')
def test_glob_src(self, stage):
@ -258,6 +266,7 @@ def test_symlinks_true(self, stage):
fs.install_tree('source', 'dest', symlinks=True)
assert os.path.exists('dest/2')
if sys.platform != "win32":
assert os.path.islink('dest/2')
check_added_exe_permissions('source/2', 'dest/2')
@ -268,6 +277,7 @@ def test_symlinks_false(self, stage):
fs.install_tree('source', 'dest', symlinks=False)
assert os.path.exists('dest/2')
if sys.platform != "win32":
assert not os.path.islink('dest/2')
check_added_exe_permissions('source/2', 'dest/2')

3
lib/spack/spack/test/llnl/util/link_tree.py
View file

@ -9,6 +9,7 @@
from llnl.util.filesystem import mkdirp, touchp, working_dir
from llnl.util.link_tree import LinkTree
from llnl.util.symlink import islink
from spack.stage import Stage
@ -42,7 +43,7 @@ def link_tree(stage):
def check_file_link(filename, expected_target):
assert os.path.isfile(filename)
assert os.path.islink(filename)
assert islink(filename)
assert (os.path.abspath(os.path.realpath(filename)) ==
os.path.abspath(expected_target))

7
lib/spack/spack/test/packaging.py
View file

@ -16,6 +16,7 @@
import pytest
from llnl.util.filesystem import mkdirp
from llnl.util.symlink import symlink
import spack.binary_distribution as bindist
import spack.cmd.buildcache as buildcache
@ -79,7 +80,7 @@ def test_buildcache(mock_archive, tmpdir):
# Create an absolute symlink
linkname = os.path.join(spec.prefix, "link_to_dummy.txt")
os.symlink(filename, linkname)
symlink(filename, linkname)
# Create the build cache and
# put it directly into the mirror
@ -232,8 +233,8 @@ def test_relocate_links(tmpdir):
with open(new_binname, 'w') as f:
f.write('\n')
os.utime(new_binname, None)
os.symlink(old_binname, new_linkname)
os.symlink('/usr/lib/libc.so', new_linkname2)
symlink(old_binname, new_linkname)
symlink('/usr/lib/libc.so', new_linkname2)
relocate_links(filenames, old_layout_root,
old_install_prefix, new_install_prefix)
assert os.readlink(new_linkname) == new_binname

7
lib/spack/spack/test/verification.py
View file

@ -8,6 +8,7 @@
import shutil
import llnl.util.filesystem as fs
from llnl.util.symlink import symlink
import spack.spec
import spack.store
@ -21,7 +22,7 @@ def test_link_manifest_entry(tmpdir):
file = str(tmpdir.join('file'))
open(file, 'a').close()
link = str(tmpdir.join('link'))
os.symlink(file, link)
symlink(file, link)
data = spack.verify.create_manifest_entry(link)
assert data['type'] == 'link'
@ -43,7 +44,7 @@ def test_link_manifest_entry(tmpdir):
file2 = str(tmpdir.join('file2'))
open(file2, 'a').close()
os.remove(link)
os.symlink(file2, link)
symlink(file2, link)
results = spack.verify.check_entry(link, data)
assert results.has_errors()
@ -157,7 +158,7 @@ def test_check_prefix_manifest(tmpdir):
f.write("I'm a little file short and stout")
link = os.path.join(bin_dir, 'run')
os.symlink(file, link)
symlink(file, link)
spack.verify.write_manifest(spec)
results = spack.verify.check_spec_manifest(spec)