environment.py: allow link:run (#29336)
* environment.py: allow link:run
Some users want minimal views, excluding run-type dependencies, since
those type of dependencies are covered by rpaths and the symlinked
libraries in the view aren't used anyways.
With this change, an environment like this:
```
spack:
specs: ['py-flake8']
view:
default:
root: view
link: run
```
includes python packages and python, but no link type deps of python.
This commit is contained in:
Harmen Stoppels
GitHub
parent
bedc9fe665
commit
dc78f4c58a
6 changed files with 148 additions and 87 deletions
|
|
@ -740,9 +740,10 @@ file snippet we define a view named ``mpis``, rooted at
|
|||
version, and compiler name to determine the path for a given
|
||||
package. This view selects all packages that depend on MPI, and
|
||||
excludes those built with the PGI compiler at version 18.5.
|
||||
All the dependencies of each root spec in the environment will be linked
|
||||
in the view due to the command ``link: all`` and the files in the view will
|
||||
be symlinks to the spack install directories.
|
||||
The root specs with their (transitive) link and run type dependencies
|
||||
will be put in the view due to the ``link: all`` option,
|
||||
and the files in the view will be symlinks to the spack install
|
||||
directories.
|
||||
|
||||
.. code-block:: yaml
|
||||
|
||||
|
|
@ -762,9 +763,22 @@ For more information on using view projections, see the section on
|
|||
:ref:`adding_projections_to_views`. The default for the ``select`` and
|
||||
``exclude`` values is to select everything and exclude nothing. The
|
||||
default projection is the default view projection (``{}``). The ``link``
|
||||
defaults to ``all`` but can also be ``roots`` when only the root specs
|
||||
in the environment are desired in the view. The ``link_type`` defaults
|
||||
to ``symlink`` but can also take the value of ``hardlink`` or ``copy``.
|
||||
attribute allows the following values:
|
||||
|
||||
#. ``link: all`` include root specs with their transitive run and link type
|
||||
dependencies (default);
|
||||
#. ``link: run`` include root specs with their transitive run type dependencies;
|
||||
#. ``link: roots`` include root specs without their dependencies.
|
||||
|
||||
The ``link_type`` defaults to ``symlink`` but can also take the value
|
||||
of ``hardlink`` or ``copy``.
|
||||
|
||||
.. tip::
|
||||
|
||||
The option ``link: run`` can be used to create small environment views for
|
||||
Python packages. Python will be able to import packages *inside* of the view even
|
||||
when the environment is not activated, and linked libraries will be located
|
||||
*outside* of the view thanks to rpaths.
|
||||
|
||||
Any number of views may be defined under the ``view`` heading in a
|
||||
Spack Environment.
|
||||
|
|
|
|||
|
|
@ -589,20 +589,31 @@ def match(string):
|
|||
return match
|
||||
|
||||
|
||||
def dedupe(sequence):
|
||||
"""Yields a stable de-duplication of an hashable sequence
|
||||
def dedupe(sequence, key=None):
|
||||
"""Yields a stable de-duplication of an hashable sequence by key
|
||||
|
||||
Args:
|
||||
sequence: hashable sequence to be de-duplicated
|
||||
key: callable applied on values before uniqueness test; identity
|
||||
by default.
|
||||
|
||||
Returns:
|
||||
stable de-duplication of the sequence
|
||||
|
||||
Examples:
|
||||
|
||||
Dedupe a list of integers:
|
||||
|
||||
[x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3]
|
||||
|
||||
[x for x in llnl.util.lang.dedupe([1,-2,1,3,2], key=abs)] == [1, -2, 3]
|
||||
"""
|
||||
seen = set()
|
||||
for x in sequence:
|
||||
if x not in seen:
|
||||
x_key = x if key is None else key(x)
|
||||
if x_key not in seen:
|
||||
yield x
|
||||
seen.add(x)
|
||||
seen.add(x_key)
|
||||
|
||||
|
||||
def pretty_date(time, now=None):
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
|
||||
import llnl.util.filesystem as fs
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.lang import dedupe
|
||||
|
||||
import spack.bootstrap
|
||||
import spack.compilers
|
||||
|
|
@ -471,80 +472,80 @@ def __contains__(self, spec):
|
|||
|
||||
return True
|
||||
|
||||
def specs_for_view(self, all_specs, roots):
|
||||
specs_for_view = []
|
||||
specs = all_specs if self.link == 'all' else roots
|
||||
def specs_for_view(self, concretized_specs):
|
||||
"""
|
||||
From the list of concretized user specs in the environment, flatten
|
||||
the dags, and filter selected, installed specs, remove duplicates on dag hash.
|
||||
"""
|
||||
specs = []
|
||||
|
||||
for spec in specs:
|
||||
# The view does not store build deps, so if we want it to
|
||||
# recognize environment specs (which do store build deps),
|
||||
# then they need to be stripped.
|
||||
if spec.concrete: # Do not link unconcretized roots
|
||||
# We preserve _hash _normal to avoid recomputing DAG
|
||||
# hashes (DAG hashes don't consider build deps)
|
||||
spec_copy = spec.copy(deps=('link', 'run'))
|
||||
spec_copy._hash = spec._hash
|
||||
spec_copy._normal = spec._normal
|
||||
specs_for_view.append(spec_copy)
|
||||
return specs_for_view
|
||||
for (_, s) in concretized_specs:
|
||||
if self.link == 'all':
|
||||
specs.extend(s.traverse(deptype=('link', 'run')))
|
||||
elif self.link == 'run':
|
||||
specs.extend(s.traverse(deptype=('run')))
|
||||
else:
|
||||
specs.append(s)
|
||||
|
||||
def regenerate(self, all_specs, roots):
|
||||
specs_for_view = self.specs_for_view(all_specs, roots)
|
||||
# De-dupe by dag hash
|
||||
specs = dedupe(specs, key=lambda s: s.dag_hash())
|
||||
|
||||
# regeneration queries the database quite a bit; this read
|
||||
# transaction ensures that we don't repeatedly lock/unlock.
|
||||
# Filter selected, installed specs
|
||||
with spack.store.db.read_transaction():
|
||||
installed_specs_for_view = set(
|
||||
s for s in specs_for_view if s in self and s.package.installed)
|
||||
specs = [s for s in specs if s in self and s.package.installed]
|
||||
|
||||
# To ensure there are no conflicts with packages being installed
|
||||
# that cannot be resolved or have repos that have been removed
|
||||
# we always regenerate the view from scratch.
|
||||
# We will do this by hashing the view contents and putting the view
|
||||
# in a directory by hash, and then having a symlink to the real
|
||||
# view in the root. The real root for a view at /dirname/basename
|
||||
# will be /dirname/._basename_<hash>.
|
||||
# This allows for atomic swaps when we update the view
|
||||
return specs
|
||||
|
||||
# cache the roots because the way we determine which is which does
|
||||
# not work while we are updating
|
||||
new_root = self._next_root(installed_specs_for_view)
|
||||
old_root = self._current_root
|
||||
def regenerate(self, concretized_specs):
|
||||
specs = self.specs_for_view(concretized_specs)
|
||||
|
||||
if new_root == old_root:
|
||||
tty.debug("View at %s does not need regeneration." % self.root)
|
||||
return
|
||||
# To ensure there are no conflicts with packages being installed
|
||||
# that cannot be resolved or have repos that have been removed
|
||||
# we always regenerate the view from scratch.
|
||||
# We will do this by hashing the view contents and putting the view
|
||||
# in a directory by hash, and then having a symlink to the real
|
||||
# view in the root. The real root for a view at /dirname/basename
|
||||
# will be /dirname/._basename_<hash>.
|
||||
# This allows for atomic swaps when we update the view
|
||||
|
||||
# construct view at new_root
|
||||
tty.msg("Updating view at {0}".format(self.root))
|
||||
# cache the roots because the way we determine which is which does
|
||||
# not work while we are updating
|
||||
new_root = self._next_root(specs)
|
||||
old_root = self._current_root
|
||||
|
||||
view = self.view(new=new_root)
|
||||
fs.mkdirp(new_root)
|
||||
view.add_specs(*installed_specs_for_view,
|
||||
with_dependencies=False)
|
||||
if new_root == old_root:
|
||||
tty.debug("View at %s does not need regeneration." % self.root)
|
||||
return
|
||||
|
||||
# create symlink from tmpname to new_root
|
||||
root_dirname = os.path.dirname(self.root)
|
||||
tmp_symlink_name = os.path.join(root_dirname, '._view_link')
|
||||
if os.path.exists(tmp_symlink_name):
|
||||
os.unlink(tmp_symlink_name)
|
||||
os.symlink(new_root, tmp_symlink_name)
|
||||
# construct view at new_root
|
||||
tty.msg("Updating view at {0}".format(self.root))
|
||||
|
||||
# mv symlink atomically over root symlink to old_root
|
||||
if os.path.exists(self.root) and not os.path.islink(self.root):
|
||||
msg = "Cannot create view: "
|
||||
msg += "file already exists and is not a link: %s" % self.root
|
||||
raise SpackEnvironmentViewError(msg)
|
||||
os.rename(tmp_symlink_name, self.root)
|
||||
view = self.view(new=new_root)
|
||||
fs.mkdirp(new_root)
|
||||
view.add_specs(*specs, with_dependencies=False)
|
||||
|
||||
# remove old_root
|
||||
if old_root and os.path.exists(old_root):
|
||||
try:
|
||||
shutil.rmtree(old_root)
|
||||
except (IOError, OSError) as e:
|
||||
msg = "Failed to remove old view at %s\n" % old_root
|
||||
msg += str(e)
|
||||
tty.warn(msg)
|
||||
# create symlink from tmpname to new_root
|
||||
root_dirname = os.path.dirname(self.root)
|
||||
tmp_symlink_name = os.path.join(root_dirname, '._view_link')
|
||||
if os.path.exists(tmp_symlink_name):
|
||||
os.unlink(tmp_symlink_name)
|
||||
os.symlink(new_root, tmp_symlink_name)
|
||||
|
||||
# mv symlink atomically over root symlink to old_root
|
||||
if os.path.exists(self.root) and not os.path.islink(self.root):
|
||||
msg = "Cannot create view: "
|
||||
msg += "file already exists and is not a link: %s" % self.root
|
||||
raise SpackEnvironmentViewError(msg)
|
||||
os.rename(tmp_symlink_name, self.root)
|
||||
|
||||
# remove old_root
|
||||
if old_root and os.path.exists(old_root):
|
||||
try:
|
||||
shutil.rmtree(old_root)
|
||||
except (IOError, OSError) as e:
|
||||
msg = "Failed to remove old view at %s\n" % old_root
|
||||
msg += str(e)
|
||||
tty.warn(msg)
|
||||
|
||||
|
||||
def _create_environment(*args, **kwargs):
|
||||
|
|
@ -1303,9 +1304,8 @@ def regenerate_views(self):
|
|||
" maintain a view")
|
||||
return
|
||||
|
||||
specs = self._get_environment_specs()
|
||||
for view in self.views.values():
|
||||
view.regenerate(specs, self.roots())
|
||||
view.regenerate(self.concretized_specs())
|
||||
|
||||
def check_views(self):
|
||||
"""Checks if the environments default view can be activated."""
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@
|
|||
},
|
||||
'link': {
|
||||
'type': 'string',
|
||||
'pattern': '(roots|all)',
|
||||
'pattern': '(roots|all|run)',
|
||||
},
|
||||
'link_type': {
|
||||
'type': 'string'
|
||||
|
|
|
|||
|
|
@ -386,23 +386,23 @@ def test_environment_status(capsys, tmpdir):
|
|||
|
||||
def test_env_status_broken_view(
|
||||
mutable_mock_env_path, mock_archive, mock_fetch, mock_packages,
|
||||
install_mockery
|
||||
install_mockery, tmpdir
|
||||
):
|
||||
with ev.create('test'):
|
||||
env_dir = str(tmpdir)
|
||||
with ev.Environment(env_dir):
|
||||
install('trivial-install-test-package')
|
||||
|
||||
# switch to a new repo that doesn't include the installed package
|
||||
# test that Spack detects the missing package and warns the user
|
||||
new_repo = MockPackageMultiRepo()
|
||||
with spack.repo.use_repositories(new_repo):
|
||||
# switch to a new repo that doesn't include the installed package
|
||||
# test that Spack detects the missing package and warns the user
|
||||
with spack.repo.use_repositories(MockPackageMultiRepo()):
|
||||
with ev.Environment(env_dir):
|
||||
output = env('status')
|
||||
assert 'In environment test' in output
|
||||
assert 'Environment test includes out of date' in output
|
||||
assert 'includes out of date packages or repos' in output
|
||||
|
||||
# Test that the warning goes away when it's fixed
|
||||
# Test that the warning goes away when it's fixed
|
||||
with ev.Environment(env_dir):
|
||||
output = env('status')
|
||||
assert 'In environment test' in output
|
||||
assert 'Environment test includes out of date' not in output
|
||||
assert 'includes out of date packages or repos' not in output
|
||||
|
||||
|
||||
def test_env_activate_broken_view(
|
||||
|
|
@ -1962,6 +1962,37 @@ def test_view_link_roots(tmpdir, mock_fetch, mock_packages, mock_archive,
|
|||
(spec.version, spec.compiler.name)))
|
||||
|
||||
|
||||
def test_view_link_run(tmpdir, mock_fetch, mock_packages, mock_archive,
|
||||
install_mockery):
|
||||
yaml = str(tmpdir.join('spack.yaml'))
|
||||
viewdir = str(tmpdir.join('view'))
|
||||
envdir = str(tmpdir)
|
||||
with open(yaml, 'w') as f:
|
||||
f.write("""
|
||||
spack:
|
||||
specs:
|
||||
- dttop
|
||||
|
||||
view:
|
||||
combinatorial:
|
||||
root: %s
|
||||
link: run
|
||||
projections:
|
||||
all: '{name}'""" % viewdir)
|
||||
|
||||
with ev.Environment(envdir):
|
||||
install()
|
||||
|
||||
# make sure transitive run type deps are in the view
|
||||
for pkg in ('dtrun1', 'dtrun3'):
|
||||
assert os.path.exists(os.path.join(viewdir, pkg))
|
||||
|
||||
# and non-run-type deps are not.
|
||||
for pkg in ('dtlink1', 'dtlink2', 'dtlink3', 'dtlink4', 'dtlink5'
|
||||
'dtbuild1', 'dtbuild2', 'dtbuild3'):
|
||||
assert not os.path.exists(os.path.join(viewdir, pkg))
|
||||
|
||||
|
||||
@pytest.mark.parametrize('link_type', ['hardlink', 'copy', 'symlink'])
|
||||
def test_view_link_type(link_type, tmpdir, mock_fetch, mock_packages, mock_archive,
|
||||
install_mockery):
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@
|
|||
import pytest
|
||||
|
||||
import llnl.util.lang
|
||||
from llnl.util.lang import match_predicate, memoized, pretty_date, stable_args
|
||||
from llnl.util.lang import dedupe, match_predicate, memoized, pretty_date, stable_args
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
|
|
@ -265,3 +265,8 @@ def f(*args, **kwargs):
|
|||
key = stable_args(*args, **kwargs)
|
||||
assert str(key) in exc_msg
|
||||
assert "function 'f'" in exc_msg
|
||||
|
||||
|
||||
def test_dedupe():
|
||||
assert [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3]
|
||||
assert [x for x in dedupe([1, -2, 1, 3, 2], key=abs)] == [1, -2, 3]
|
||||
|
|
|
|||
Loading…
Reference in a new issue