Updates to symlinking for Windows (#34701)
* Support hardlinks/junctions on Windows systems without developer
mode enabled
* Generally, use of llnl.util.symlink.symlink is preferred over
os.symlink since it handles this automatically
* Generally an error is now reported if a user attempts to create a
symlink to a file that does not exist (this was previously allowed
on Linux/Mac).
* One exception to this: when Spack installs files from the source
into their final prefix, dangling symlinks are allowed (on
Linux/Mac - Windows does not allow this in any circumstance).
The intent behind this is to avoid generating failures for
installations on Linux/Mac that were succeeding before.
* Because Windows is strict about forbidding dangling symlinks,
`traverse_tree` has been updated to skip creating symlinks if they
would point to a file that is ignored. This check is not
transitive (i.e., a symlink to a symlink to an ignored file would
not be caught appropriately)
* Relocate function: resolve_link_target_relative_to_the_link
(this is not otherwise modified)
Co-authored-by: jamessmillie <smillie@txcorp.com>
This commit is contained in:
David Alexander
GitHub
parent
5db241c755
commit
66f75407d1
13 changed files with 817 additions and 191 deletions
|
|
@ -18,11 +18,13 @@
|
|||
import sys
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from itertools import accumulate
|
||||
from typing import Callable, Iterable, List, Match, Optional, Tuple, Union
|
||||
|
||||
import llnl.util.symlink
|
||||
from llnl.util import tty
|
||||
from llnl.util.lang import dedupe, memoized
|
||||
from llnl.util.symlink import islink, symlink
|
||||
from llnl.util.symlink import islink, readlink, resolve_link_target_relative_to_the_link, symlink
|
||||
|
||||
from spack.util.executable import Executable, which
|
||||
from spack.util.path import path_to_os_path, system_path_filter
|
||||
|
|
@ -101,7 +103,7 @@ def _nop(args, ns=None, follow_symlinks=None):
|
|||
pass
|
||||
|
||||
# follow symlinks (aka don't not follow symlinks)
|
||||
follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst))
|
||||
follow = follow_symlinks or not (islink(src) and islink(dst))
|
||||
if follow:
|
||||
# use the real function if it exists
|
||||
def lookup(name):
|
||||
|
|
@ -169,7 +171,7 @@ def rename(src, dst):
|
|||
if sys.platform == "win32":
|
||||
# Windows path existence checks will sometimes fail on junctions/links/symlinks
|
||||
# so check for that case
|
||||
if os.path.exists(dst) or os.path.islink(dst):
|
||||
if os.path.exists(dst) or islink(dst):
|
||||
os.remove(dst)
|
||||
os.rename(src, dst)
|
||||
|
||||
|
|
@ -566,7 +568,7 @@ def set_install_permissions(path):
|
|||
# If this points to a file maintained in a Spack prefix, it is assumed that
|
||||
# this function will be invoked on the target. If the file is outside a
|
||||
# Spack-maintained prefix, the permissions should not be modified.
|
||||
if os.path.islink(path):
|
||||
if islink(path):
|
||||
return
|
||||
if os.path.isdir(path):
|
||||
os.chmod(path, 0o755)
|
||||
|
|
@ -633,7 +635,7 @@ def chmod_x(entry, perms):
|
|||
@system_path_filter
|
||||
def copy_mode(src, dest):
|
||||
"""Set the mode of dest to that of src unless it is a link."""
|
||||
if os.path.islink(dest):
|
||||
if islink(dest):
|
||||
return
|
||||
src_mode = os.stat(src).st_mode
|
||||
dest_mode = os.stat(dest).st_mode
|
||||
|
|
@ -719,26 +721,12 @@ def install(src, dest):
|
|||
copy(src, dest, _permissions=True)
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def resolve_link_target_relative_to_the_link(link):
|
||||
"""
|
||||
os.path.isdir uses os.path.exists, which for links will check
|
||||
the existence of the link target. If the link target is relative to
|
||||
the link, we need to construct a pathname that is valid from
|
||||
our cwd (which may not be the same as the link's directory)
|
||||
"""
|
||||
target = os.readlink(link)
|
||||
if os.path.isabs(target):
|
||||
return target
|
||||
link_dir = os.path.dirname(os.path.abspath(link))
|
||||
return os.path.join(link_dir, target)
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def copy_tree(
|
||||
src: str,
|
||||
dest: str,
|
||||
symlinks: bool = True,
|
||||
allow_broken_symlinks: bool = False,
|
||||
ignore: Optional[Callable[[str], bool]] = None,
|
||||
_permissions: bool = False,
|
||||
):
|
||||
|
|
@ -761,6 +749,8 @@ def copy_tree(
|
|||
src (str): the directory to copy
|
||||
dest (str): the destination directory
|
||||
symlinks (bool): whether or not to preserve symlinks
|
||||
allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks,
|
||||
On Windows, setting this to True will raise an exception.
|
||||
ignore (typing.Callable): function indicating which files to ignore
|
||||
_permissions (bool): for internal use only
|
||||
|
||||
|
|
@ -768,6 +758,8 @@ def copy_tree(
|
|||
IOError: if *src* does not match any files or directories
|
||||
ValueError: if *src* is a parent directory of *dest*
|
||||
"""
|
||||
if allow_broken_symlinks and sys.platform == "win32":
|
||||
raise llnl.util.symlink.SymlinkError("Cannot allow broken symlinks on Windows!")
|
||||
if _permissions:
|
||||
tty.debug("Installing {0} to {1}".format(src, dest))
|
||||
else:
|
||||
|
|
@ -781,6 +773,11 @@ def copy_tree(
|
|||
if not files:
|
||||
raise IOError("No such file or directory: '{0}'".format(src))
|
||||
|
||||
# For Windows hard-links and junctions, the source path must exist to make a symlink. Add
|
||||
# all symlinks to this list while traversing the tree, then when finished, make all
|
||||
# symlinks at the end.
|
||||
links = []
|
||||
|
||||
for src in files:
|
||||
abs_src = os.path.abspath(src)
|
||||
if not abs_src.endswith(os.path.sep):
|
||||
|
|
@ -803,21 +800,27 @@ def copy_tree(
|
|||
ignore=ignore,
|
||||
follow_nonexisting=True,
|
||||
):
|
||||
if os.path.islink(s):
|
||||
if islink(s):
|
||||
link_target = resolve_link_target_relative_to_the_link(s)
|
||||
if symlinks:
|
||||
target = os.readlink(s)
|
||||
if os.path.isabs(target):
|
||||
|
||||
def escaped_path(path):
|
||||
return path.replace("\\", r"\\")
|
||||
|
||||
if os.path.isabs(target):
|
||||
new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target)
|
||||
|
||||
else:
|
||||
new_target = re.sub(escaped_path(src), escaped_path(dest), target)
|
||||
|
||||
if new_target != target:
|
||||
tty.debug("Redirecting link {0} to {1}".format(target, new_target))
|
||||
target = new_target
|
||||
|
||||
symlink(target, d)
|
||||
links.append((target, d, s))
|
||||
continue
|
||||
|
||||
elif os.path.isdir(link_target):
|
||||
mkdirp(d)
|
||||
else:
|
||||
|
|
@ -832,9 +835,15 @@ def escaped_path(path):
|
|||
set_install_permissions(d)
|
||||
copy_mode(s, d)
|
||||
|
||||
for target, d, s in links:
|
||||
symlink(target, d, allow_broken_symlinks=allow_broken_symlinks)
|
||||
if _permissions:
|
||||
set_install_permissions(d)
|
||||
copy_mode(s, d)
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def install_tree(src, dest, symlinks=True, ignore=None):
|
||||
def install_tree(src, dest, symlinks=True, ignore=None, allow_broken_symlinks=False):
|
||||
"""Recursively install an entire directory tree rooted at *src*.
|
||||
|
||||
Same as :py:func:`copy_tree` with the addition of setting proper
|
||||
|
|
@ -845,12 +854,21 @@ def install_tree(src, dest, symlinks=True, ignore=None):
|
|||
dest (str): the destination directory
|
||||
symlinks (bool): whether or not to preserve symlinks
|
||||
ignore (typing.Callable): function indicating which files to ignore
|
||||
allow_broken_symlinks (bool): whether or not to allow broken (dangling) symlinks,
|
||||
On Windows, setting this to True will raise an exception.
|
||||
|
||||
Raises:
|
||||
IOError: if *src* does not match any files or directories
|
||||
ValueError: if *src* is a parent directory of *dest*
|
||||
"""
|
||||
copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)
|
||||
copy_tree(
|
||||
src,
|
||||
dest,
|
||||
symlinks=symlinks,
|
||||
allow_broken_symlinks=allow_broken_symlinks,
|
||||
ignore=ignore,
|
||||
_permissions=True,
|
||||
)
|
||||
|
||||
|
||||
@system_path_filter
|
||||
|
|
@ -1254,7 +1272,12 @@ def traverse_tree(
|
|||
Keyword Arguments:
|
||||
order (str): Whether to do pre- or post-order traversal. Accepted
|
||||
values are 'pre' and 'post'
|
||||
ignore (typing.Callable): function indicating which files to ignore
|
||||
ignore (typing.Callable): function indicating which files to ignore. This will also
|
||||
ignore symlinks if they point to an ignored file (regardless of whether the symlink
|
||||
is explicitly ignored); note this only supports one layer of indirection (i.e. if
|
||||
you have x -> y -> z, and z is ignored but x/y are not, then y would be ignored
|
||||
but not x). To avoid this, make sure the ignore function also ignores the symlink
|
||||
paths too.
|
||||
follow_nonexisting (bool): Whether to descend into directories in
|
||||
``src`` that do not exit in ``dest``. Default is True
|
||||
follow_links (bool): Whether to descend into symlinks in ``src``
|
||||
|
|
@ -1281,11 +1304,22 @@ def traverse_tree(
|
|||
dest_child = os.path.join(dest_path, f)
|
||||
rel_child = os.path.join(rel_path, f)
|
||||
|
||||
# If the source path is a link and the link's source is ignored, then ignore the link too.
|
||||
if islink(source_child) and not follow_links:
|
||||
target = readlink(source_child)
|
||||
all_parents = accumulate(target.split(os.sep), lambda x, y: os.path.join(x, y))
|
||||
if any(map(ignore, all_parents)):
|
||||
tty.warn(
|
||||
f"Skipping {source_path} because the source or a part of the source's "
|
||||
f"path is included in the ignores."
|
||||
)
|
||||
continue
|
||||
|
||||
# Treat as a directory
|
||||
# TODO: for symlinks, os.path.isdir looks for the link target. If the
|
||||
# target is relative to the link, then that may not resolve properly
|
||||
# relative to our cwd - see resolve_link_target_relative_to_the_link
|
||||
if os.path.isdir(source_child) and (follow_links or not os.path.islink(source_child)):
|
||||
if os.path.isdir(source_child) and (follow_links or not islink(source_child)):
|
||||
# When follow_nonexisting isn't set, don't descend into dirs
|
||||
# in source that do not exist in dest
|
||||
if follow_nonexisting or os.path.exists(dest_child):
|
||||
|
|
@ -1311,7 +1345,12 @@ def traverse_tree(
|
|||
|
||||
def lexists_islink_isdir(path):
|
||||
"""Computes the tuple (lexists(path), islink(path), isdir(path)) in a minimal
|
||||
number of stat calls."""
|
||||
number of stat calls on unix. Use os.path and symlink.islink methods for windows."""
|
||||
if sys.platform == "win32":
|
||||
if not os.path.lexists(path):
|
||||
return False, False, False
|
||||
return os.path.lexists(path), islink(path), os.path.isdir(path)
|
||||
else:
|
||||
# First try to lstat, so we know if it's a link or not.
|
||||
try:
|
||||
lst = os.lstat(path)
|
||||
|
|
@ -1526,7 +1565,7 @@ def remove_if_dead_link(path):
|
|||
Parameters:
|
||||
path (str): The potential dead link
|
||||
"""
|
||||
if os.path.islink(path) and not os.path.exists(path):
|
||||
if islink(path) and not os.path.exists(path):
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
|
|
@ -1585,7 +1624,7 @@ def remove_linked_tree(path):
|
|||
kwargs["onerror"] = readonly_file_handler(ignore_errors=True)
|
||||
|
||||
if os.path.exists(path):
|
||||
if os.path.islink(path):
|
||||
if islink(path):
|
||||
shutil.rmtree(os.path.realpath(path), **kwargs)
|
||||
os.unlink(path)
|
||||
else:
|
||||
|
|
@ -2686,7 +2725,7 @@ def remove_directory_contents(dir):
|
|||
"""Remove all contents of a directory."""
|
||||
if os.path.exists(dir):
|
||||
for entry in [os.path.join(dir, entry) for entry in os.listdir(dir)]:
|
||||
if os.path.isfile(entry) or os.path.islink(entry):
|
||||
if os.path.isfile(entry) or islink(entry):
|
||||
os.unlink(entry)
|
||||
else:
|
||||
shutil.rmtree(entry)
|
||||
|
|
|
|||
|
|
@ -2,77 +2,185 @@
|
|||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
import errno
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from os.path import exists, join
|
||||
|
||||
from llnl.util import lang
|
||||
from llnl.util import lang, tty
|
||||
|
||||
from spack.error import SpackError
|
||||
from spack.util.path import system_path_filter
|
||||
|
||||
if sys.platform == "win32":
|
||||
from win32file import CreateHardLink
|
||||
|
||||
|
||||
def symlink(real_path, link_path):
|
||||
def symlink(source_path: str, link_path: str, allow_broken_symlinks: bool = False):
|
||||
"""
|
||||
Create a symbolic link.
|
||||
Create a link.
|
||||
|
||||
On Windows, use junctions if os.symlink fails.
|
||||
On non-Windows and Windows with System Administrator
|
||||
privleges this will be a normal symbolic link via
|
||||
os.symlink.
|
||||
|
||||
On Windows without privledges the link will be a
|
||||
junction for a directory and a hardlink for a file.
|
||||
On Windows the various link types are:
|
||||
|
||||
Symbolic Link: A link to a file or directory on the
|
||||
same or different volume (drive letter) or even to
|
||||
a remote file or directory (using UNC in its path).
|
||||
Need System Administrator privileges to make these.
|
||||
|
||||
Hard Link: A link to a file on the same volume (drive
|
||||
letter) only. Every file (file's data) has at least 1
|
||||
hard link (file's name). But when this method creates
|
||||
a new hard link there will be 2. Deleting all hard
|
||||
links effectively deletes the file. Don't need System
|
||||
Administrator privileges.
|
||||
|
||||
Junction: A link to a directory on the same or different
|
||||
volume (drive letter) but not to a remote directory. Don't
|
||||
need System Administrator privileges.
|
||||
|
||||
Parameters:
|
||||
source_path (str): The real file or directory that the link points to.
|
||||
Must be absolute OR relative to the link.
|
||||
link_path (str): The path where the link will exist.
|
||||
allow_broken_symlinks (bool): On Linux or Mac, don't raise an exception if the source_path
|
||||
doesn't exist. This will still raise an exception on Windows.
|
||||
"""
|
||||
if sys.platform != "win32":
|
||||
os.symlink(real_path, link_path)
|
||||
elif _win32_can_symlink():
|
||||
# Windows requires target_is_directory=True when the target is a dir.
|
||||
os.symlink(real_path, link_path, target_is_directory=os.path.isdir(real_path))
|
||||
source_path = os.path.normpath(source_path)
|
||||
win_source_path = source_path
|
||||
link_path = os.path.normpath(link_path)
|
||||
|
||||
# Never allow broken links on Windows.
|
||||
if sys.platform == "win32" and allow_broken_symlinks:
|
||||
raise ValueError("allow_broken_symlinks parameter cannot be True on Windows.")
|
||||
|
||||
# Perform basic checks to make sure symlinking will succeed
|
||||
if os.path.lexists(link_path):
|
||||
raise SymlinkError(f"Link path ({link_path}) already exists. Cannot create link.")
|
||||
|
||||
if not os.path.exists(source_path):
|
||||
if os.path.isabs(source_path) and not allow_broken_symlinks:
|
||||
# An absolute source path that does not exist will result in a broken link.
|
||||
raise SymlinkError(
|
||||
f"Source path ({source_path}) is absolute but does not exist. Resulting "
|
||||
f"link would be broken so not making link."
|
||||
)
|
||||
else:
|
||||
try:
|
||||
# Try to use junctions
|
||||
_win32_junction(real_path, link_path)
|
||||
except OSError as e:
|
||||
if e.errno == errno.EEXIST:
|
||||
# EEXIST error indicates that file we're trying to "link"
|
||||
# is already present, don't bother trying to copy which will also fail
|
||||
# just raise
|
||||
raise
|
||||
# os.symlink can create a link when the given source path is relative to
|
||||
# the link path. Emulate this behavior and check to see if the source exists
|
||||
# relative to the link patg ahead of link creation to prevent broken
|
||||
# links from being made.
|
||||
link_parent_dir = os.path.dirname(link_path)
|
||||
relative_path = os.path.join(link_parent_dir, source_path)
|
||||
if os.path.exists(relative_path):
|
||||
# In order to work on windows, the source path needs to be modified to be
|
||||
# relative because hardlink/junction dont resolve relative paths the same way as
|
||||
# os.symlink. This is ignored on other operating systems.
|
||||
win_source_path = relative_path
|
||||
elif not allow_broken_symlinks:
|
||||
raise SymlinkError(
|
||||
f"The source path ({source_path}) is not relative to the link path "
|
||||
f"({link_path}). Resulting link would be broken so not making link."
|
||||
)
|
||||
|
||||
# Create the symlink
|
||||
if sys.platform == "win32" and not _windows_can_symlink():
|
||||
_windows_create_link(win_source_path, link_path)
|
||||
else:
|
||||
# If all else fails, fall back to copying files
|
||||
shutil.copyfile(real_path, link_path)
|
||||
os.symlink(source_path, link_path, target_is_directory=os.path.isdir(source_path))
|
||||
|
||||
|
||||
def islink(path):
|
||||
return os.path.islink(path) or _win32_is_junction(path)
|
||||
def islink(path: str) -> bool:
|
||||
"""Override os.islink to give correct answer for spack logic.
|
||||
|
||||
For Non-Windows: a link can be determined with the os.path.islink method.
|
||||
Windows-only methods will return false for other operating systems.
|
||||
|
||||
For Windows: spack considers symlinks, hard links, and junctions to
|
||||
all be links, so if any of those are True, return True.
|
||||
|
||||
Args:
|
||||
path (str): path to check if it is a link.
|
||||
|
||||
Returns:
|
||||
bool - whether the path is any kind link or not.
|
||||
"""
|
||||
return any([os.path.islink(path), _windows_is_junction(path), _windows_is_hardlink(path)])
|
||||
|
||||
|
||||
# '_win32' functions based on
|
||||
# https://github.com/Erotemic/ubelt/blob/master/ubelt/util_links.py
|
||||
def _win32_junction(path, link):
|
||||
# junctions require absolute paths
|
||||
if not os.path.isabs(link):
|
||||
link = os.path.abspath(link)
|
||||
def _windows_is_hardlink(path: str) -> bool:
|
||||
"""Determines if a path is a windows hard link. This is accomplished
|
||||
by looking at the number of links using os.stat. A non-hard-linked file
|
||||
will have a st_nlink value of 1, whereas a hard link will have a value
|
||||
larger than 1. Note that both the original and hard-linked file will
|
||||
return True because they share the same inode.
|
||||
|
||||
# os.symlink will fail if link exists, emulate the behavior here
|
||||
if exists(link):
|
||||
raise OSError(errno.EEXIST, "File exists: %s -> %s" % (link, path))
|
||||
Args:
|
||||
path (str): Windows path to check for a hard link
|
||||
|
||||
if not os.path.isabs(path):
|
||||
parent = os.path.join(link, os.pardir)
|
||||
path = os.path.join(parent, path)
|
||||
path = os.path.abspath(path)
|
||||
Returns:
|
||||
bool - Whether the path is a hard link or not.
|
||||
"""
|
||||
if sys.platform != "win32" or os.path.islink(path) or not os.path.exists(path):
|
||||
return False
|
||||
|
||||
CreateHardLink(link, path)
|
||||
return os.stat(path).st_nlink > 1
|
||||
|
||||
|
||||
def _windows_is_junction(path: str) -> bool:
|
||||
"""Determines if a path is a windows junction. A junction can be
|
||||
determined using a bitwise AND operation between the file's
|
||||
attribute bitmask and the known junction bitmask (0x400).
|
||||
|
||||
Args:
|
||||
path (str): A non-file path
|
||||
|
||||
Returns:
|
||||
bool - whether the path is a junction or not.
|
||||
"""
|
||||
if sys.platform != "win32" or os.path.islink(path) or os.path.isfile(path):
|
||||
return False
|
||||
|
||||
import ctypes.wintypes
|
||||
|
||||
get_file_attributes = ctypes.windll.kernel32.GetFileAttributesW # type: ignore[attr-defined]
|
||||
get_file_attributes.argtypes = (ctypes.wintypes.LPWSTR,)
|
||||
get_file_attributes.restype = ctypes.wintypes.DWORD
|
||||
|
||||
invalid_file_attributes = 0xFFFFFFFF
|
||||
reparse_point = 0x400
|
||||
file_attr = get_file_attributes(path)
|
||||
|
||||
if file_attr == invalid_file_attributes:
|
||||
return False
|
||||
|
||||
return file_attr & reparse_point > 0
|
||||
|
||||
|
||||
@lang.memoized
|
||||
def _win32_can_symlink():
|
||||
def _windows_can_symlink() -> bool:
|
||||
"""
|
||||
Determines if windows is able to make a symlink depending on
|
||||
the system configuration and the level of the user's permissions.
|
||||
"""
|
||||
if sys.platform != "win32":
|
||||
tty.warn("windows_can_symlink method can't be used on non-Windows OS.")
|
||||
return False
|
||||
|
||||
tempdir = tempfile.mkdtemp()
|
||||
|
||||
dpath = join(tempdir, "dpath")
|
||||
fpath = join(tempdir, "fpath.txt")
|
||||
dpath = os.path.join(tempdir, "dpath")
|
||||
fpath = os.path.join(tempdir, "fpath.txt")
|
||||
|
||||
dlink = join(tempdir, "dlink")
|
||||
flink = join(tempdir, "flink.txt")
|
||||
dlink = os.path.join(tempdir, "dlink")
|
||||
flink = os.path.join(tempdir, "flink.txt")
|
||||
|
||||
import llnl.util.filesystem as fs
|
||||
|
||||
|
|
@ -96,24 +204,136 @@ def _win32_can_symlink():
|
|||
return can_symlink_directories and can_symlink_files
|
||||
|
||||
|
||||
def _win32_is_junction(path):
|
||||
def _windows_create_link(source: str, link: str):
|
||||
"""
|
||||
Determines if a path is a win32 junction
|
||||
Attempts to create a Hard Link or Junction as an alternative
|
||||
to a symbolic link. This is called when symbolic links cannot
|
||||
be created.
|
||||
"""
|
||||
if os.path.islink(path):
|
||||
return False
|
||||
if sys.platform != "win32":
|
||||
raise SymlinkError("windows_create_link method can't be used on non-Windows OS.")
|
||||
elif os.path.isdir(source):
|
||||
_windows_create_junction(source=source, link=link)
|
||||
elif os.path.isfile(source):
|
||||
_windows_create_hard_link(path=source, link=link)
|
||||
else:
|
||||
raise SymlinkError(
|
||||
f"Cannot create link from {source}. It is neither a file nor a directory."
|
||||
)
|
||||
|
||||
if sys.platform == "win32":
|
||||
import ctypes.wintypes
|
||||
|
||||
GetFileAttributes = ctypes.windll.kernel32.GetFileAttributesW
|
||||
GetFileAttributes.argtypes = (ctypes.wintypes.LPWSTR,)
|
||||
GetFileAttributes.restype = ctypes.wintypes.DWORD
|
||||
def _windows_create_junction(source: str, link: str):
|
||||
"""Duly verify that the path and link are eligible to create a junction,
|
||||
then create the junction.
|
||||
"""
|
||||
if sys.platform != "win32":
|
||||
raise SymlinkError("windows_create_junction method can't be used on non-Windows OS.")
|
||||
elif not os.path.exists(source):
|
||||
raise SymlinkError("Source path does not exist, cannot create a junction.")
|
||||
elif os.path.lexists(link):
|
||||
raise SymlinkError("Link path already exists, cannot create a junction.")
|
||||
elif not os.path.isdir(source):
|
||||
raise SymlinkError("Source path is not a directory, cannot create a junction.")
|
||||
|
||||
INVALID_FILE_ATTRIBUTES = 0xFFFFFFFF
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 0x400
|
||||
import subprocess
|
||||
|
||||
res = GetFileAttributes(path)
|
||||
return res != INVALID_FILE_ATTRIBUTES and bool(res & FILE_ATTRIBUTE_REPARSE_POINT)
|
||||
cmd = ["cmd", "/C", "mklink", "/J", link, source]
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
out, err = proc.communicate()
|
||||
tty.debug(out.decode())
|
||||
if proc.returncode != 0:
|
||||
err = err.decode()
|
||||
tty.error(err)
|
||||
raise SymlinkError("Make junction command returned a non-zero return code.", err)
|
||||
|
||||
return False
|
||||
|
||||
def _windows_create_hard_link(path: str, link: str):
|
||||
"""Duly verify that the path and link are eligible to create a hard
|
||||
link, then create the hard link.
|
||||
"""
|
||||
if sys.platform != "win32":
|
||||
raise SymlinkError("windows_create_hard_link method can't be used on non-Windows OS.")
|
||||
elif not os.path.exists(path):
|
||||
raise SymlinkError(f"File path {path} does not exist. Cannot create hard link.")
|
||||
elif os.path.lexists(link):
|
||||
raise SymlinkError(f"Link path ({link}) already exists. Cannot create hard link.")
|
||||
elif not os.path.isfile(path):
|
||||
raise SymlinkError(f"File path ({link}) is not a file. Cannot create hard link.")
|
||||
else:
|
||||
tty.debug(f"Creating hard link {link} pointing to {path}")
|
||||
CreateHardLink(link, path)
|
||||
|
||||
|
||||
def readlink(path: str):
|
||||
"""Spack utility to override of os.readlink method to work cross platform"""
|
||||
if _windows_is_hardlink(path):
|
||||
return _windows_read_hard_link(path)
|
||||
elif _windows_is_junction(path):
|
||||
return _windows_read_junction(path)
|
||||
else:
|
||||
return os.readlink(path)
|
||||
|
||||
|
||||
def _windows_read_hard_link(link: str) -> str:
|
||||
"""Find all of the files that point to the same inode as the link"""
|
||||
if sys.platform != "win32":
|
||||
raise SymlinkError("Can't read hard link on non-Windows OS.")
|
||||
link = os.path.abspath(link)
|
||||
fsutil_cmd = ["fsutil", "hardlink", "list", link]
|
||||
proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
out, err = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
raise SymlinkError(f"An error occurred while reading hard link: {err.decode()}")
|
||||
|
||||
# fsutil response does not include the drive name, so append it back to each linked file.
|
||||
drive, link_tail = os.path.splitdrive(os.path.abspath(link))
|
||||
links = set([os.path.join(drive, p) for p in out.decode().splitlines()])
|
||||
links.remove(link)
|
||||
if len(links) == 1:
|
||||
return links.pop()
|
||||
elif len(links) > 1:
|
||||
# TODO: How best to handle the case where 3 or more paths point to a single inode?
|
||||
raise SymlinkError(f"Found multiple paths pointing to the same inode {links}")
|
||||
else:
|
||||
raise SymlinkError("Cannot determine hard link source path.")
|
||||
|
||||
|
||||
def _windows_read_junction(link: str):
|
||||
"""Find the path that a junction points to."""
|
||||
if sys.platform != "win32":
|
||||
raise SymlinkError("Can't read junction on non-Windows OS.")
|
||||
|
||||
link = os.path.abspath(link)
|
||||
link_basename = os.path.basename(link)
|
||||
link_parent = os.path.dirname(link)
|
||||
fsutil_cmd = ["dir", "/a:l", link_parent]
|
||||
proc = subprocess.Popen(fsutil_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
|
||||
out, err = proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
raise SymlinkError(f"An error occurred while reading junction: {err.decode()}")
|
||||
matches = re.search(rf"<JUNCTION>\s+{link_basename} \[(.*)]", out.decode())
|
||||
if matches:
|
||||
return matches.group(1)
|
||||
else:
|
||||
raise SymlinkError("Could not find junction path.")
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def resolve_link_target_relative_to_the_link(link):
|
||||
"""
|
||||
os.path.isdir uses os.path.exists, which for links will check
|
||||
the existence of the link target. If the link target is relative to
|
||||
the link, we need to construct a pathname that is valid from
|
||||
our cwd (which may not be the same as the link's directory)
|
||||
"""
|
||||
target = readlink(link)
|
||||
if os.path.isabs(target):
|
||||
return target
|
||||
link_dir = os.path.dirname(os.path.abspath(link))
|
||||
return os.path.join(link_dir, target)
|
||||
|
||||
|
||||
class SymlinkError(SpackError):
|
||||
"""Exception class for errors raised while creating symlinks,
|
||||
junctions and hard links
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@
|
|||
import shutil
|
||||
|
||||
import llnl.util.lang
|
||||
import llnl.util.symlink as symlink
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.symlink import symlink
|
||||
|
||||
import spack.compiler
|
||||
import spack.compilers.clang
|
||||
|
|
@ -216,11 +216,11 @@ def setup_custom_environment(self, pkg, env):
|
|||
for fname in os.listdir(dev_dir):
|
||||
if fname in bins:
|
||||
os.unlink(os.path.join(dev_dir, fname))
|
||||
symlink(
|
||||
symlink.symlink(
|
||||
os.path.join(spack.paths.build_env_path, "cc"),
|
||||
os.path.join(dev_dir, fname),
|
||||
)
|
||||
|
||||
symlink(developer_root, xcode_link)
|
||||
symlink.symlink(developer_root, xcode_link)
|
||||
|
||||
env.set("DEVELOPER_DIR", xcode_link)
|
||||
|
|
|
|||
|
|
@ -174,6 +174,7 @@ def _do_fake_install(pkg):
|
|||
# Install fake command
|
||||
fs.mkdirp(pkg.prefix.bin)
|
||||
fs.touch(os.path.join(pkg.prefix.bin, command))
|
||||
fs.touchp(pkg.install_log_path)
|
||||
if sys.platform != "win32":
|
||||
chmod = which("chmod")
|
||||
chmod("+x", os.path.join(pkg.prefix.bin, command))
|
||||
|
|
@ -1984,7 +1985,9 @@ def _install_source(self):
|
|||
src_target = os.path.join(pkg.spec.prefix, "share", pkg.name, "src")
|
||||
tty.debug("{0} Copying source to {1}".format(self.pre, src_target))
|
||||
|
||||
fs.install_tree(pkg.stage.source_path, src_target)
|
||||
fs.install_tree(
|
||||
pkg.stage.source_path, src_target, allow_broken_symlinks=(sys.platform != "win32")
|
||||
)
|
||||
|
||||
def _real_install(self):
|
||||
import spack.builder
|
||||
|
|
|
|||
|
|
@ -229,6 +229,7 @@ def test_concretize_target_ranges(root_target_range, dep_target_range, result, m
|
|||
(["21.11", "21.9"], None, False),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Cray does not use windows")
|
||||
def test_cray_platform_detection(versions, default, expected, tmpdir, monkeypatch, working_env):
|
||||
ex_path = str(tmpdir.join("fake_craype_dir"))
|
||||
fs.mkdirp(ex_path)
|
||||
|
|
|
|||
|
|
@ -31,13 +31,16 @@ def test_fetch_missing_cache(tmpdir, _fetch_method):
|
|||
@pytest.mark.parametrize("_fetch_method", ["curl", "urllib"])
|
||||
def test_fetch(tmpdir, _fetch_method):
|
||||
"""Ensure a fetch after expanding is effectively a no-op."""
|
||||
testpath = str(tmpdir)
|
||||
cache = os.path.join(testpath, "cache.tar.gz")
|
||||
cache_dir = tmpdir.join("cache")
|
||||
stage_dir = tmpdir.join("stage")
|
||||
mkdirp(cache_dir)
|
||||
mkdirp(stage_dir)
|
||||
cache = os.path.join(cache_dir, "cache.tar.gz")
|
||||
touch(cache)
|
||||
url = url_util.path_to_file_url(cache)
|
||||
with spack.config.override("config:url_fetch_method", _fetch_method):
|
||||
fetcher = CacheURLFetchStrategy(url=url)
|
||||
with Stage(fetcher, path=testpath) as stage:
|
||||
with Stage(fetcher, path=str(stage_dir)) as stage:
|
||||
source_path = stage.source_path
|
||||
mkdirp(source_path)
|
||||
fetcher.fetch()
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
import pytest
|
||||
|
||||
import llnl.util.filesystem as fs
|
||||
import llnl.util.symlink as symlink
|
||||
|
||||
import spack.compiler
|
||||
import spack.compilers as compilers
|
||||
|
|
@ -841,9 +842,12 @@ def test_apple_clang_setup_environment(mock_executable, monkeypatch):
|
|||
class MockPackage(object):
|
||||
use_xcode = False
|
||||
|
||||
apple_clang_cls = spack.compilers.class_for_compiler_name("apple-clang")
|
||||
apple_clang = "apple-clang"
|
||||
apple_clang_version = "11.0.0"
|
||||
|
||||
apple_clang_cls = spack.compilers.class_for_compiler_name(apple_clang)
|
||||
compiler = apple_clang_cls(
|
||||
spack.spec.CompilerSpec("apple-clang@=11.0.0"),
|
||||
spack.spec.CompilerSpec(f"{apple_clang}@={apple_clang_version}"),
|
||||
"catalina",
|
||||
"x86_64",
|
||||
["/usr/bin/clang", "/usr/bin/clang++", None, None],
|
||||
|
|
@ -889,7 +893,7 @@ def _listdir(path):
|
|||
# Set a few operations to noop
|
||||
monkeypatch.setattr(shutil, "copytree", noop)
|
||||
monkeypatch.setattr(os, "unlink", noop)
|
||||
monkeypatch.setattr(os, "symlink", noop)
|
||||
monkeypatch.setattr(symlink, "symlink", noop)
|
||||
monkeypatch.setattr(os, "listdir", _listdir)
|
||||
|
||||
# Qt is so far the only package that uses this code path, change
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@
|
|||
import llnl.util.lang
|
||||
import llnl.util.lock
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, working_dir
|
||||
from llnl.util.filesystem import copy_tree, mkdirp, remove_linked_tree, touchp, working_dir
|
||||
|
||||
import spack.binary_distribution
|
||||
import spack.caches
|
||||
|
|
@ -565,6 +565,8 @@ def mock_repo_path():
|
|||
def _pkg_install_fn(pkg, spec, prefix):
|
||||
# sanity_check_prefix requires something in the install directory
|
||||
mkdirp(prefix.bin)
|
||||
if not os.path.exists(spec.package.build_log_path):
|
||||
touchp(spec.package.build_log_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
import pytest
|
||||
|
||||
import llnl.util.filesystem as fs
|
||||
from llnl.util.symlink import islink, symlink
|
||||
from llnl.util.symlink import SymlinkError, _windows_can_symlink, islink, symlink
|
||||
|
||||
import spack.paths
|
||||
|
||||
|
|
@ -150,7 +150,6 @@ def test_multiple_src_file_dest(self, stage):
|
|||
fs.install("source/a/*/*", "dest/1")
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Skip test on Windows")
|
||||
class TestCopyTree:
|
||||
"""Tests for ``filesystem.copy_tree``"""
|
||||
|
||||
|
|
@ -189,7 +188,7 @@ def test_symlinks_true(self, stage):
|
|||
def test_symlinks_true_ignore(self, stage):
|
||||
"""Test copying when specifying relative paths that should be ignored"""
|
||||
with fs.working_dir(str(stage)):
|
||||
ignore = lambda p: p in ["c/d/e", "a"]
|
||||
ignore = lambda p: p in [os.path.join("c", "d", "e"), "a"]
|
||||
fs.copy_tree("source", "dest", symlinks=True, ignore=ignore)
|
||||
assert not os.path.exists("dest/a")
|
||||
assert os.path.exists("dest/c/d")
|
||||
|
|
@ -231,7 +230,6 @@ def test_parent_dir(self, stage):
|
|||
fs.copy_tree("source", "source/sub/directory")
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Skip test on Windows")
|
||||
class TestInstallTree:
|
||||
"""Tests for ``filesystem.install_tree``"""
|
||||
|
||||
|
|
@ -275,6 +273,15 @@ def test_symlinks_false(self, stage):
|
|||
assert not os.path.islink("dest/2")
|
||||
check_added_exe_permissions("source/2", "dest/2")
|
||||
|
||||
@pytest.mark.skipif(sys.platform == "win32", reason="Broken symlinks not allowed on Windows")
|
||||
def test_allow_broken_symlinks(self, stage):
|
||||
"""Test installing with a broken symlink."""
|
||||
with fs.working_dir(str(stage)):
|
||||
symlink("nonexistant.txt", "source/broken", allow_broken_symlinks=True)
|
||||
fs.install_tree("source", "dest", symlinks=True, allow_broken_symlinks=True)
|
||||
assert os.path.islink("dest/broken")
|
||||
assert not os.path.exists(os.readlink("dest/broken"))
|
||||
|
||||
def test_glob_src(self, stage):
|
||||
"""Test using a glob as the source."""
|
||||
|
||||
|
|
@ -712,6 +719,9 @@ def test_is_nonsymlink_exe_with_shebang(tmpdir):
|
|||
assert not fs.is_nonsymlink_exe_with_shebang("symlink_to_executable_script")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32" and not _windows_can_symlink(), reason="Requires elevated privileges."
|
||||
)
|
||||
def test_lexists_islink_isdir(tmpdir):
|
||||
root = str(tmpdir)
|
||||
|
||||
|
|
@ -730,19 +740,69 @@ def test_lexists_islink_isdir(tmpdir):
|
|||
with open(file, "wb") as f:
|
||||
f.write(b"file")
|
||||
|
||||
os.symlink("dir", symlink_to_dir)
|
||||
os.symlink("file", symlink_to_file)
|
||||
os.symlink("does_not_exist", dangling_symlink)
|
||||
os.symlink("dangling_symlink", symlink_to_dangling_symlink)
|
||||
os.symlink("symlink_to_dir", symlink_to_symlink_to_dir)
|
||||
os.symlink("symlink_to_file", symlink_to_symlink_to_file)
|
||||
symlink("dir", symlink_to_dir)
|
||||
symlink("file", symlink_to_file)
|
||||
|
||||
with pytest.raises(SymlinkError):
|
||||
symlink("does_not_exist", dangling_symlink)
|
||||
symlink("dangling_symlink", symlink_to_dangling_symlink)
|
||||
|
||||
symlink("symlink_to_dir", symlink_to_symlink_to_dir)
|
||||
symlink("symlink_to_file", symlink_to_symlink_to_file)
|
||||
|
||||
assert fs.lexists_islink_isdir(dir) == (True, False, True)
|
||||
assert fs.lexists_islink_isdir(file) == (True, False, False)
|
||||
assert fs.lexists_islink_isdir(nonexistent) == (False, False, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_dir) == (True, True, True)
|
||||
assert fs.lexists_islink_isdir(symlink_to_file) == (True, True, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (True, True, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (False, False, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_symlink_to_dir) == (True, True, True)
|
||||
assert fs.lexists_islink_isdir(symlink_to_symlink_to_file) == (True, True, False)
|
||||
|
||||
|
||||
@pytest.mark.skipif(_windows_can_symlink(), reason="Not to be run with elevated privileges.")
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="For Windows Only")
|
||||
def test_lexists_islink_isdir__win32_base(tmpdir):
|
||||
"""Run on windows without elevated privileges to test junctions and hard links which have
|
||||
different results from the lexists_islink_isdir method.
|
||||
"""
|
||||
root = str(tmpdir)
|
||||
|
||||
# Create a directory and a file, an a bunch of symlinks.
|
||||
dir = os.path.join(root, "dir")
|
||||
file = os.path.join(root, "file")
|
||||
nonexistent = os.path.join(root, "does_not_exist")
|
||||
symlink_to_dir = os.path.join(root, "symlink_to_dir")
|
||||
symlink_to_file = os.path.join(root, "symlink_to_file")
|
||||
dangling_symlink = os.path.join(root, "dangling_symlink")
|
||||
symlink_to_dangling_symlink = os.path.join(root, "symlink_to_dangling_symlink")
|
||||
symlink_to_symlink_to_dir = os.path.join(root, "symlink_to_symlink_to_dir")
|
||||
symlink_to_symlink_to_file = os.path.join(root, "symlink_to_symlink_to_file")
|
||||
|
||||
os.mkdir(dir)
|
||||
assert fs.lexists_islink_isdir(dir) == (True, False, True)
|
||||
|
||||
symlink("dir", symlink_to_dir)
|
||||
assert fs.lexists_islink_isdir(dir) == (True, False, True)
|
||||
assert fs.lexists_islink_isdir(symlink_to_dir) == (True, True, True)
|
||||
|
||||
with open(file, "wb") as f:
|
||||
f.write(b"file")
|
||||
assert fs.lexists_islink_isdir(file) == (True, False, False)
|
||||
|
||||
symlink("file", symlink_to_file)
|
||||
assert fs.lexists_islink_isdir(file) == (True, True, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_file) == (True, True, False)
|
||||
|
||||
with pytest.raises(SymlinkError):
|
||||
symlink("does_not_exist", dangling_symlink)
|
||||
symlink("dangling_symlink", symlink_to_dangling_symlink)
|
||||
|
||||
symlink("symlink_to_dir", symlink_to_symlink_to_dir)
|
||||
symlink("symlink_to_file", symlink_to_symlink_to_file)
|
||||
|
||||
assert fs.lexists_islink_isdir(nonexistent) == (False, False, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_dangling_symlink) == (False, False, False)
|
||||
assert fs.lexists_islink_isdir(symlink_to_symlink_to_dir) == (True, True, True)
|
||||
assert fs.lexists_islink_isdir(symlink_to_symlink_to_file) == (True, True, False)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,12 +4,14 @@
|
|||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import llnl.util.symlink
|
||||
from llnl.util.filesystem import mkdirp, touchp, visit_directory_tree, working_dir
|
||||
from llnl.util.link_tree import DestinationMergeVisitor, LinkTree, SourceMergeVisitor
|
||||
from llnl.util.symlink import islink
|
||||
from llnl.util.symlink import _windows_can_symlink, islink, readlink, symlink
|
||||
|
||||
from spack.stage import Stage
|
||||
|
||||
|
|
@ -44,6 +46,7 @@ def link_tree(stage):
|
|||
def check_file_link(filename, expected_target):
|
||||
assert os.path.isfile(filename)
|
||||
assert islink(filename)
|
||||
if sys.platform != "win32" or llnl.util.symlink._windows_can_symlink():
|
||||
assert os.path.abspath(os.path.realpath(filename)) == os.path.abspath(expected_target)
|
||||
|
||||
|
||||
|
|
@ -51,70 +54,108 @@ def check_dir(filename):
|
|||
assert os.path.isdir(filename)
|
||||
|
||||
|
||||
def test_merge_to_new_directory(stage, link_tree):
|
||||
@pytest.mark.parametrize("run_as_root", [True, False])
|
||||
def test_merge_to_new_directory(stage, link_tree, monkeypatch, run_as_root):
|
||||
if sys.platform != "win32":
|
||||
if run_as_root:
|
||||
pass
|
||||
else:
|
||||
pytest.skip("Skipping duplicate test.")
|
||||
elif _windows_can_symlink() or not run_as_root:
|
||||
monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root)
|
||||
else:
|
||||
# Skip if trying to run as dev-mode without having dev-mode.
|
||||
pytest.skip("Skipping portion of test which required dev-mode privileges.")
|
||||
|
||||
with working_dir(stage.path):
|
||||
link_tree.merge("dest")
|
||||
|
||||
check_file_link("dest/1", "source/1")
|
||||
check_file_link("dest/a/b/2", "source/a/b/2")
|
||||
check_file_link("dest/a/b/3", "source/a/b/3")
|
||||
check_file_link("dest/c/4", "source/c/4")
|
||||
check_file_link("dest/c/d/5", "source/c/d/5")
|
||||
check_file_link("dest/c/d/6", "source/c/d/6")
|
||||
check_file_link("dest/c/d/e/7", "source/c/d/e/7")
|
||||
files = [
|
||||
("dest/1", "source/1"),
|
||||
("dest/a/b/2", "source/a/b/2"),
|
||||
("dest/a/b/3", "source/a/b/3"),
|
||||
("dest/c/4", "source/c/4"),
|
||||
("dest/c/d/5", "source/c/d/5"),
|
||||
("dest/c/d/6", "source/c/d/6"),
|
||||
("dest/c/d/e/7", "source/c/d/e/7"),
|
||||
]
|
||||
|
||||
assert os.path.isabs(os.readlink("dest/1"))
|
||||
assert os.path.isabs(os.readlink("dest/a/b/2"))
|
||||
assert os.path.isabs(os.readlink("dest/a/b/3"))
|
||||
assert os.path.isabs(os.readlink("dest/c/4"))
|
||||
assert os.path.isabs(os.readlink("dest/c/d/5"))
|
||||
assert os.path.isabs(os.readlink("dest/c/d/6"))
|
||||
assert os.path.isabs(os.readlink("dest/c/d/e/7"))
|
||||
for dest, source in files:
|
||||
check_file_link(dest, source)
|
||||
assert os.path.isabs(readlink(dest))
|
||||
|
||||
link_tree.unmerge("dest")
|
||||
|
||||
assert not os.path.exists("dest")
|
||||
|
||||
|
||||
def test_merge_to_new_directory_relative(stage, link_tree):
|
||||
@pytest.mark.parametrize("run_as_root", [True, False])
|
||||
def test_merge_to_new_directory_relative(stage, link_tree, monkeypatch, run_as_root):
|
||||
if sys.platform != "win32":
|
||||
if run_as_root:
|
||||
pass
|
||||
else:
|
||||
pytest.skip("Skipping duplicate test.")
|
||||
elif _windows_can_symlink() or not run_as_root:
|
||||
monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root)
|
||||
else:
|
||||
# Skip if trying to run as dev-mode without having dev-mode.
|
||||
pytest.skip("Skipping portion of test which required dev-mode privileges.")
|
||||
|
||||
with working_dir(stage.path):
|
||||
link_tree.merge("dest", relative=True)
|
||||
|
||||
check_file_link("dest/1", "source/1")
|
||||
check_file_link("dest/a/b/2", "source/a/b/2")
|
||||
check_file_link("dest/a/b/3", "source/a/b/3")
|
||||
check_file_link("dest/c/4", "source/c/4")
|
||||
check_file_link("dest/c/d/5", "source/c/d/5")
|
||||
check_file_link("dest/c/d/6", "source/c/d/6")
|
||||
check_file_link("dest/c/d/e/7", "source/c/d/e/7")
|
||||
files = [
|
||||
("dest/1", "source/1"),
|
||||
("dest/a/b/2", "source/a/b/2"),
|
||||
("dest/a/b/3", "source/a/b/3"),
|
||||
("dest/c/4", "source/c/4"),
|
||||
("dest/c/d/5", "source/c/d/5"),
|
||||
("dest/c/d/6", "source/c/d/6"),
|
||||
("dest/c/d/e/7", "source/c/d/e/7"),
|
||||
]
|
||||
|
||||
assert not os.path.isabs(os.readlink("dest/1"))
|
||||
assert not os.path.isabs(os.readlink("dest/a/b/2"))
|
||||
assert not os.path.isabs(os.readlink("dest/a/b/3"))
|
||||
assert not os.path.isabs(os.readlink("dest/c/4"))
|
||||
assert not os.path.isabs(os.readlink("dest/c/d/5"))
|
||||
assert not os.path.isabs(os.readlink("dest/c/d/6"))
|
||||
assert not os.path.isabs(os.readlink("dest/c/d/e/7"))
|
||||
for dest, source in files:
|
||||
check_file_link(dest, source)
|
||||
# Hard links/junctions are inherently absolute.
|
||||
if sys.platform != "win32" or run_as_root:
|
||||
assert not os.path.isabs(readlink(dest))
|
||||
|
||||
link_tree.unmerge("dest")
|
||||
|
||||
assert not os.path.exists("dest")
|
||||
|
||||
|
||||
def test_merge_to_existing_directory(stage, link_tree):
|
||||
@pytest.mark.parametrize("run_as_root", [True, False])
|
||||
def test_merge_to_existing_directory(stage, link_tree, monkeypatch, run_as_root):
|
||||
if sys.platform != "win32":
|
||||
if run_as_root:
|
||||
pass
|
||||
else:
|
||||
pytest.skip("Skipping duplicate test.")
|
||||
elif _windows_can_symlink() or not run_as_root:
|
||||
monkeypatch.setattr(llnl.util.symlink, "_windows_can_symlink", lambda: run_as_root)
|
||||
else:
|
||||
# Skip if trying to run as dev-mode without having dev-mode.
|
||||
pytest.skip("Skipping portion of test which required dev-mode privileges.")
|
||||
|
||||
with working_dir(stage.path):
|
||||
touchp("dest/x")
|
||||
touchp("dest/a/b/y")
|
||||
|
||||
link_tree.merge("dest")
|
||||
|
||||
check_file_link("dest/1", "source/1")
|
||||
check_file_link("dest/a/b/2", "source/a/b/2")
|
||||
check_file_link("dest/a/b/3", "source/a/b/3")
|
||||
check_file_link("dest/c/4", "source/c/4")
|
||||
check_file_link("dest/c/d/5", "source/c/d/5")
|
||||
check_file_link("dest/c/d/6", "source/c/d/6")
|
||||
check_file_link("dest/c/d/e/7", "source/c/d/e/7")
|
||||
files = [
|
||||
("dest/1", "source/1"),
|
||||
("dest/a/b/2", "source/a/b/2"),
|
||||
("dest/a/b/3", "source/a/b/3"),
|
||||
("dest/c/4", "source/c/4"),
|
||||
("dest/c/d/5", "source/c/d/5"),
|
||||
("dest/c/d/6", "source/c/d/6"),
|
||||
("dest/c/d/e/7", "source/c/d/e/7"),
|
||||
]
|
||||
for dest, source in files:
|
||||
check_file_link(dest, source)
|
||||
|
||||
assert os.path.isfile("dest/x")
|
||||
assert os.path.isfile("dest/a/b/y")
|
||||
|
|
@ -124,13 +165,8 @@ def test_merge_to_existing_directory(stage, link_tree):
|
|||
assert os.path.isfile("dest/x")
|
||||
assert os.path.isfile("dest/a/b/y")
|
||||
|
||||
assert not os.path.isfile("dest/1")
|
||||
assert not os.path.isfile("dest/a/b/2")
|
||||
assert not os.path.isfile("dest/a/b/3")
|
||||
assert not os.path.isfile("dest/c/4")
|
||||
assert not os.path.isfile("dest/c/d/5")
|
||||
assert not os.path.isfile("dest/c/d/6")
|
||||
assert not os.path.isfile("dest/c/d/e/7")
|
||||
for dest, _ in files:
|
||||
assert not os.path.isfile(dest)
|
||||
|
||||
|
||||
def test_merge_with_empty_directories(stage, link_tree):
|
||||
|
|
@ -192,9 +228,9 @@ def test_source_merge_visitor_does_not_follow_symlinked_dirs_at_depth(tmpdir):
|
|||
os.mkdir(j("a", "b"))
|
||||
os.mkdir(j("a", "b", "c"))
|
||||
os.mkdir(j("a", "b", "c", "d"))
|
||||
os.symlink(j("b"), j("a", "symlink_b"))
|
||||
os.symlink(j("c"), j("a", "b", "symlink_c"))
|
||||
os.symlink(j("d"), j("a", "b", "c", "symlink_d"))
|
||||
symlink(j("b"), j("a", "symlink_b"))
|
||||
symlink(j("c"), j("a", "b", "symlink_c"))
|
||||
symlink(j("d"), j("a", "b", "c", "symlink_d"))
|
||||
with open(j("a", "b", "c", "d", "file"), "wb"):
|
||||
pass
|
||||
|
||||
|
|
@ -236,10 +272,11 @@ def test_source_merge_visitor_cant_be_cyclical(tmpdir):
|
|||
j = os.path.join
|
||||
with tmpdir.as_cwd():
|
||||
os.mkdir(j("a"))
|
||||
os.symlink(j("..", "b"), j("a", "symlink_b"))
|
||||
os.symlink(j("symlink_b"), j("a", "symlink_b_b"))
|
||||
os.mkdir(j("b"))
|
||||
os.symlink(j("..", "a"), j("b", "symlink_a"))
|
||||
|
||||
symlink(j("..", "b"), j("a", "symlink_b"))
|
||||
symlink(j("symlink_b"), j("a", "symlink_b_b"))
|
||||
symlink(j("..", "a"), j("b", "symlink_a"))
|
||||
|
||||
visitor = SourceMergeVisitor()
|
||||
visit_directory_tree(str(tmpdir), visitor)
|
||||
|
|
|
|||
246
lib/spack/spack/test/llnl/util/symlink.py
Normal file
246
lib/spack/spack/test/llnl/util/symlink.py
Normal file
|
|
@ -0,0 +1,246 @@
|
|||
# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
"""Tests for ``llnl/util/symlink.py``"""
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from llnl.util import symlink
|
||||
|
||||
|
||||
def test_symlink_file(tmpdir):
|
||||
"""Test the symlink.symlink functionality on all operating systems for a file"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir)
|
||||
link_file = str(tmpdir.join("link.txt"))
|
||||
assert os.path.exists(link_file) is False
|
||||
symlink.symlink(source_path=real_file, link_path=link_file)
|
||||
assert os.path.exists(link_file)
|
||||
assert symlink.islink(link_file)
|
||||
|
||||
|
||||
def test_symlink_dir(tmpdir):
|
||||
"""Test the symlink.symlink functionality on all operating systems for a directory"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
real_dir = os.path.join(test_dir, "real_dir")
|
||||
link_dir = os.path.join(test_dir, "link_dir")
|
||||
os.mkdir(real_dir)
|
||||
symlink.symlink(source_path=real_dir, link_path=link_dir)
|
||||
assert os.path.exists(link_dir)
|
||||
assert symlink.islink(link_dir)
|
||||
|
||||
|
||||
def test_symlink_source_not_exists(tmpdir):
|
||||
"""Test the symlink.symlink method for the case where a source path does not exist"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
real_dir = os.path.join(test_dir, "real_dir")
|
||||
link_dir = os.path.join(test_dir, "link_dir")
|
||||
with pytest.raises(symlink.SymlinkError):
|
||||
symlink.symlink(source_path=real_dir, link_path=link_dir)
|
||||
|
||||
|
||||
def test_symlink_src_relative_to_link(tmpdir):
|
||||
"""Test the symlink.symlink functionality where the source value exists relative to the link
|
||||
but not relative to the cwd"""
|
||||
with tmpdir.as_cwd():
|
||||
subdir_1 = tmpdir.join("a")
|
||||
subdir_2 = os.path.join(subdir_1, "b")
|
||||
link_dir = os.path.join(subdir_1, "c")
|
||||
|
||||
os.mkdir(subdir_1)
|
||||
os.mkdir(subdir_2)
|
||||
|
||||
fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2)
|
||||
link_file = os.path.join(subdir_1, "link.txt")
|
||||
|
||||
symlink.symlink(
|
||||
source_path=f"b/{os.path.basename(real_file)}",
|
||||
link_path=f"a/{os.path.basename(link_file)}",
|
||||
)
|
||||
assert os.path.exists(link_file)
|
||||
assert symlink.islink(link_file)
|
||||
# Check dirs
|
||||
assert not os.path.lexists(link_dir)
|
||||
symlink.symlink(source_path="b", link_path="a/c")
|
||||
assert os.path.lexists(link_dir)
|
||||
|
||||
|
||||
def test_symlink_src_not_relative_to_link(tmpdir):
|
||||
"""Test the symlink.symlink functionality where the source value does not exist relative to
|
||||
the link and not relative to the cwd. NOTE that this symlink api call is EXPECTED to raise
|
||||
a symlink.SymlinkError exception that we catch."""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
subdir_1 = os.path.join(test_dir, "a")
|
||||
subdir_2 = os.path.join(subdir_1, "b")
|
||||
link_dir = os.path.join(subdir_1, "c")
|
||||
os.mkdir(subdir_1)
|
||||
os.mkdir(subdir_2)
|
||||
fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=subdir_2)
|
||||
link_file = str(tmpdir.join("link.txt"))
|
||||
# Expected SymlinkError because source path does not exist relative to link path
|
||||
with pytest.raises(symlink.SymlinkError):
|
||||
symlink.symlink(
|
||||
source_path=f"d/{os.path.basename(real_file)}",
|
||||
link_path=f"a/{os.path.basename(link_file)}",
|
||||
)
|
||||
assert not os.path.exists(link_file)
|
||||
# Check dirs
|
||||
assert not os.path.lexists(link_dir)
|
||||
with pytest.raises(symlink.SymlinkError):
|
||||
symlink.symlink(source_path="d", link_path="a/c")
|
||||
assert not os.path.lexists(link_dir)
|
||||
|
||||
|
||||
def test_symlink_link_already_exists(tmpdir):
|
||||
"""Test the symlink.symlink method for the case where a link already exists"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
real_dir = os.path.join(test_dir, "real_dir")
|
||||
link_dir = os.path.join(test_dir, "link_dir")
|
||||
os.mkdir(real_dir)
|
||||
symlink.symlink(real_dir, link_dir)
|
||||
assert os.path.exists(link_dir)
|
||||
with pytest.raises(symlink.SymlinkError):
|
||||
symlink.symlink(source_path=real_dir, link_path=link_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not symlink._windows_can_symlink(), reason="Test requires elevated privileges")
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_symlink_win_file(tmpdir):
|
||||
"""Check that symlink.symlink makes a symlink file when run with elevated permissions"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir)
|
||||
link_file = str(tmpdir.join("link.txt"))
|
||||
symlink.symlink(source_path=real_file, link_path=link_file)
|
||||
# Verify that all expected conditions are met
|
||||
assert os.path.exists(link_file)
|
||||
assert symlink.islink(link_file)
|
||||
assert os.path.islink(link_file)
|
||||
assert not symlink._windows_is_hardlink(link_file)
|
||||
assert not symlink._windows_is_junction(link_file)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not symlink._windows_can_symlink(), reason="Test requires elevated privileges")
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_symlink_win_dir(tmpdir):
|
||||
"""Check that symlink.symlink makes a symlink dir when run with elevated permissions"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
real_dir = os.path.join(test_dir, "real")
|
||||
link_dir = os.path.join(test_dir, "link")
|
||||
os.mkdir(real_dir)
|
||||
symlink.symlink(source_path=real_dir, link_path=link_dir)
|
||||
# Verify that all expected conditions are met
|
||||
assert os.path.exists(link_dir)
|
||||
assert symlink.islink(link_dir)
|
||||
assert os.path.islink(link_dir)
|
||||
assert not symlink._windows_is_hardlink(link_dir)
|
||||
assert not symlink._windows_is_junction(link_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_windows_create_junction(tmpdir):
|
||||
"""Test the symlink._windows_create_junction method"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
junction_real_dir = os.path.join(test_dir, "real_dir")
|
||||
junction_link_dir = os.path.join(test_dir, "link_dir")
|
||||
os.mkdir(junction_real_dir)
|
||||
symlink._windows_create_junction(junction_real_dir, junction_link_dir)
|
||||
# Verify that all expected conditions are met
|
||||
assert os.path.exists(junction_link_dir)
|
||||
assert symlink._windows_is_junction(junction_link_dir)
|
||||
assert symlink.islink(junction_link_dir)
|
||||
assert not os.path.islink(junction_link_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_windows_create_hard_link(tmpdir):
|
||||
"""Test the symlink._windows_create_hard_link method"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir)
|
||||
link_file = str(tmpdir.join("link.txt"))
|
||||
symlink._windows_create_hard_link(real_file, link_file)
|
||||
# Verify that all expected conditions are met
|
||||
assert os.path.exists(link_file)
|
||||
assert symlink._windows_is_hardlink(real_file)
|
||||
assert symlink._windows_is_hardlink(link_file)
|
||||
assert symlink.islink(link_file)
|
||||
assert not os.path.islink(link_file)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_windows_create_link_dir(tmpdir):
|
||||
"""Test the functionality of the windows_create_link method with a directory
|
||||
which should result in making a junction.
|
||||
"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
real_dir = os.path.join(test_dir, "real")
|
||||
link_dir = os.path.join(test_dir, "link")
|
||||
os.mkdir(real_dir)
|
||||
symlink._windows_create_link(real_dir, link_dir)
|
||||
# Verify that all expected conditions are met
|
||||
assert os.path.exists(link_dir)
|
||||
assert symlink.islink(link_dir)
|
||||
assert not symlink._windows_is_hardlink(link_dir)
|
||||
assert symlink._windows_is_junction(link_dir)
|
||||
assert not os.path.islink(link_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_windows_create_link_file(tmpdir):
|
||||
"""Test the functionality of the windows_create_link method with a file
|
||||
which should result in the creation of a hard link. It also tests the
|
||||
functionality of the symlink islink infrastructure.
|
||||
"""
|
||||
with tmpdir.as_cwd():
|
||||
test_dir = str(tmpdir)
|
||||
fd, real_file = tempfile.mkstemp(prefix="real", suffix=".txt", dir=test_dir)
|
||||
link_file = str(tmpdir.join("link.txt"))
|
||||
symlink._windows_create_link(real_file, link_file)
|
||||
# Verify that all expected conditions are met
|
||||
assert symlink._windows_is_hardlink(link_file)
|
||||
assert symlink.islink(link_file)
|
||||
assert not symlink._windows_is_junction(link_file)
|
||||
|
||||
|
||||
@pytest.mark.skipif(sys.platform != "win32", reason="Test is only for Windows")
|
||||
def test_windows_read_link(tmpdir):
|
||||
"""Makes sure symlink.readlink can read the link source for hard links and
|
||||
junctions on windows."""
|
||||
with tmpdir.as_cwd():
|
||||
real_dir_1 = "real_dir_1"
|
||||
real_dir_2 = "real_dir_2"
|
||||
link_dir_1 = "link_dir_1"
|
||||
link_dir_2 = "link_dir_2"
|
||||
os.mkdir(real_dir_1)
|
||||
os.mkdir(real_dir_2)
|
||||
|
||||
# Create a file and a directory
|
||||
_, real_file_1 = tempfile.mkstemp(prefix="real_1", suffix=".txt", dir=".")
|
||||
_, real_file_2 = tempfile.mkstemp(prefix="real_2", suffix=".txt", dir=".")
|
||||
link_file_1 = "link_1.txt"
|
||||
link_file_2 = "link_2.txt"
|
||||
|
||||
# Make hard link/junction
|
||||
symlink._windows_create_hard_link(real_file_1, link_file_1)
|
||||
symlink._windows_create_hard_link(real_file_2, link_file_2)
|
||||
symlink._windows_create_junction(real_dir_1, link_dir_1)
|
||||
symlink._windows_create_junction(real_dir_2, link_dir_2)
|
||||
|
||||
assert symlink.readlink(link_file_1) == os.path.abspath(real_file_1)
|
||||
assert symlink.readlink(link_file_2) == os.path.abspath(real_file_2)
|
||||
assert symlink.readlink(link_dir_1) == os.path.abspath(real_dir_1)
|
||||
assert symlink.readlink(link_dir_2) == os.path.abspath(real_dir_2)
|
||||
|
|
@ -9,7 +9,7 @@
|
|||
|
||||
import pytest
|
||||
|
||||
from llnl.util.filesystem import resolve_link_target_relative_to_the_link
|
||||
from llnl.util.symlink import resolve_link_target_relative_to_the_link
|
||||
|
||||
import spack.mirror
|
||||
import spack.repo
|
||||
|
|
@ -224,6 +224,9 @@ def successful_expand(_class):
|
|||
def successful_apply(*args, **kwargs):
|
||||
pass
|
||||
|
||||
def successful_symlink(*args, **kwargs):
|
||||
pass
|
||||
|
||||
with Stage("spack-mirror-test") as stage:
|
||||
mirror_root = os.path.join(stage.path, "test-mirror")
|
||||
|
||||
|
|
@ -231,6 +234,7 @@ def successful_apply(*args, **kwargs):
|
|||
monkeypatch.setattr(spack.fetch_strategy.URLFetchStrategy, "expand", successful_expand)
|
||||
monkeypatch.setattr(spack.patch, "apply_patch", successful_apply)
|
||||
monkeypatch.setattr(spack.caches.MirrorCache, "store", record_store)
|
||||
monkeypatch.setattr(spack.caches.MirrorCache, "symlink", successful_symlink)
|
||||
|
||||
with spack.config.override("config:checksum", False):
|
||||
spack.mirror.create(mirror_root, list(spec.traverse()))
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@
|
|||
|
||||
import pytest
|
||||
|
||||
from llnl.util.filesystem import mkdirp
|
||||
from llnl.util.filesystem import mkdirp, touchp
|
||||
from llnl.util.symlink import symlink
|
||||
|
||||
import spack.binary_distribution as bindist
|
||||
|
|
@ -188,8 +188,15 @@ def test_relocate_links(tmpdir):
|
|||
|
||||
own_prefix_path = str(tmpdir.join("prefix_a", "file"))
|
||||
dep_prefix_path = str(tmpdir.join("prefix_b", "file"))
|
||||
new_own_prefix_path = str(tmpdir.join("new_prefix_a", "file"))
|
||||
new_dep_prefix_path = str(tmpdir.join("new_prefix_b", "file"))
|
||||
system_path = os.path.join(os.path.sep, "system", "path")
|
||||
|
||||
touchp(own_prefix_path)
|
||||
touchp(new_own_prefix_path)
|
||||
touchp(dep_prefix_path)
|
||||
touchp(new_dep_prefix_path)
|
||||
|
||||
# Old prefixes to new prefixes
|
||||
prefix_to_prefix = OrderedDict(
|
||||
[
|
||||
|
|
|
|||
Loading…
Reference in a new issue