From 9faee51e22967b289dc7d8aabe0dffd36a0f85ba Mon Sep 17 00:00:00 2001 From: Massimiliano Culpo Date: Mon, 7 Oct 2019 18:53:23 +0200 Subject: [PATCH] Spack environments can concretize specs together (#11372) This PR adds a 'concretize' entry to an environment's spec.yaml file which controls how user specs are concretized. By default it is set to 'separately' which means that each spec added by the user is concretized separately (the behavior of environments before this PR). If set to 'together', the environment will concretize all of the added user specs together; this means that all specs and their dependencies will be consistent with each other (for example, a user could develop code linked against the set of libraries in the environment without conflicts). If the environment was previously concretized, this will re-concretize all specs, in which case previously-installed specs may no longer be used by the environment (in this sense, adding a new spec to an environment with 'concretize: together' can be significantly more expensive). The 'concretize: together' setting is not compatible with Spec matrices; this PR adds a check to look for multiple instances of the same package added to the environment and fails early when 'concretize: together' is set (to avoid confusing messages about conflicts later on). --- lib/spack/docs/environments.rst | 52 ++++++++++++++++++++++-- lib/spack/spack/environment.py | 70 ++++++++++++++++++++++++++++++++- lib/spack/spack/schema/env.py | 5 +++ lib/spack/spack/test/cmd/env.py | 48 ++++++++++++++++++++++ 4 files changed, 171 insertions(+), 4 deletions(-) diff --git a/lib/spack/docs/environments.rst b/lib/spack/docs/environments.rst index 6ce67f0067..1ce765210f 100644 --- a/lib/spack/docs/environments.rst +++ b/lib/spack/docs/environments.rst @@ -292,19 +292,37 @@ or $ spack -E myenv add python +.. _environments_concretization: + ^^^^^^^^^^^^ Concretizing ^^^^^^^^^^^^ Once some user specs have been added to an environment, they can be -concretized. The following command will concretize all user specs -that have been added and not yet concretized: +concretized. *By default specs are concretized separately*, one after +the other. This mode of operation permits to deploy a full +software stack where multiple configurations of the same package +need to be installed alongside each other. Central installations done +at HPC centers by system administrators or user support groups +are a common case that fits in this behavior. +Environments *can also be configured to concretize all +the root specs in a self-consistent way* to ensure that +each package in the environment comes with a single configuration. This +mode of operation is usually what is required by software developers that +want to deploy their development environment. + +Regardless of which mode of operation has been chosen, the following +command will ensure all the root specs are concretized according to the +constraints that are prescribed in the configuration: .. code-block:: console [myenv]$ spack concretize -This command will re-concretize all specs: +In the case of specs that are not concretized together, the command +above will concretize only the specs that were added and not yet +concretized. Forcing a re-concretization of all the specs can be done +instead with this command: .. code-block:: console @@ -467,6 +485,34 @@ Appending to this list in the yaml is identical to using the ``spack add`` command from the command line. However, there is more power available from the yaml file. +""""""""""""""""""" +Spec concretization +""""""""""""""""""" + +Specs can be concretized separately or together, as already +explained in :ref:`environments_concretization`. The behavior active +under any environment is determined by the ``concretization`` property: + +.. code-block:: yaml + + spack: + specs: + - ncview + - netcdf + - nco + - py-sphinx + concretization: together + +which can currently take either one of the two allowed values ``together`` or ``separately`` +(the default). + +.. admonition:: Re-concretization of user specs + + When concretizing specs together the entire set of specs will be + re-concretized after any addition of new user specs, to ensure that + the environment remains consistent. When instead the specs are concretized + separately only the new specs will be re-concretized after any addition. + """"""""""""" Spec Matrices """"""""""""" diff --git a/lib/spack/spack/environment.py b/lib/spack/spack/environment.py index 84c77df6b0..ab8d7e14f1 100644 --- a/lib/spack/spack/environment.py +++ b/lib/spack/spack/environment.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections import os import re import sys @@ -19,6 +20,7 @@ import llnl.util.tty as tty from llnl.util.tty.color import colorize +import spack.concretize import spack.error import spack.hash_types as ht import spack.repo @@ -514,6 +516,9 @@ def __init__(self, path, init_file=None, with_view=None): path to the view. """ self.path = os.path.abspath(path) + # This attribute will be set properly from configuration + # during concretization + self.concretization = None self.clear() if init_file: @@ -591,6 +596,9 @@ def _read_manifest(self, f): for name, values in enable_view.items()) else: self.views = {} + # Retrieve the current concretization strategy + configuration = config_dict(self.yaml) + self.concretization = configuration.get('concretization') @property def user_specs(self): @@ -845,6 +853,59 @@ def concretize(self, force=False): self.concretized_order = [] self.specs_by_hash = {} + # Pick the right concretization strategy + if self.concretization == 'together': + return self._concretize_together() + if self.concretization == 'separately': + return self._concretize_separately() + + msg = 'concretization strategy not implemented [{0}]' + raise SpackEnvironmentError(msg.format(self.concretization)) + + def _concretize_together(self): + """Concretization strategy that concretizes all the specs + in the same DAG. + """ + # Exit early if the set of concretized specs is the set of user specs + user_specs_did_not_change = not bool( + set(self.user_specs) - set(self.concretized_user_specs) + ) + if user_specs_did_not_change: + return [] + + # Check that user specs don't have duplicate packages + counter = collections.defaultdict(int) + for user_spec in self.user_specs: + counter[user_spec.name] += 1 + + duplicates = [] + for name, count in counter.items(): + if count > 1: + duplicates.append(name) + + if duplicates: + msg = ('environment that are configured to concretize specs' + ' together cannot contain more than one spec for each' + ' package [{0}]'.format(', '.join(duplicates))) + raise SpackEnvironmentError(msg) + + # Proceed with concretization + self.concretized_user_specs = [] + self.concretized_order = [] + self.specs_by_hash = {} + + concrete_specs = spack.concretize.concretize_specs_together( + *self.user_specs + ) + concretized_specs = [x for x in zip(self.user_specs, concrete_specs)] + for abstract, concrete in concretized_specs: + self._add_concrete_spec(abstract, concrete) + return concretized_specs + + def _concretize_separately(self): + """Concretization strategy that concretizes separately one + user spec after the other. + """ # keep any concretized specs whose user specs are still in the manifest old_concretized_user_specs = self.concretized_user_specs old_concretized_order = self.concretized_order @@ -875,6 +936,13 @@ def install(self, user_spec, concrete_spec=None, **install_args): This will automatically concretize the single spec, but it won't affect other as-yet unconcretized specs. """ + if self.concretization == 'together': + msg = 'cannot install a single spec in an environment that is ' \ + 'configured to be concretized together. Run instead:\n\n' \ + ' $ spack add \n' \ + ' $ spack install\n' + raise SpackEnvironmentError(msg) + spec = Spec(user_spec) if self.add(spec): @@ -1292,7 +1360,7 @@ def write(self): def __enter__(self): self._previous_active = _active_environment activate(self) - return + return self def __exit__(self, exc_type, exc_val, exc_tb): deactivate() diff --git a/lib/spack/spack/schema/env.py b/lib/spack/spack/schema/env.py index 2a3afb0aef..0af877185a 100644 --- a/lib/spack/spack/schema/env.py +++ b/lib/spack/spack/schema/env.py @@ -119,6 +119,11 @@ } } ] + }, + 'concretization': { + 'type': 'string', + 'enum': ['together', 'separately'], + 'default': 'separately' } } ) diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 45b409df39..b7f3e5e325 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -1694,3 +1694,51 @@ def test_env_activate_csh_prints_shell_output( assert "setenv SPACK_ENV" in out assert "set prompt=" in out assert "alias despacktivate" in out + + +def test_concretize_user_specs_together(): + e = ev.create('coconcretization') + e.concretization = 'together' + + # Concretize a first time using 'mpich' as the MPI provider + e.add('mpileaks') + e.add('mpich') + e.concretize() + + assert all('mpich' in spec for _, spec in e.concretized_specs()) + assert all('mpich2' not in spec for _, spec in e.concretized_specs()) + + # Concretize a second time using 'mpich2' as the MPI provider + e.remove('mpich') + e.add('mpich2') + e.concretize() + + assert all('mpich2' in spec for _, spec in e.concretized_specs()) + assert all('mpich' not in spec for _, spec in e.concretized_specs()) + + # Concretize again without changing anything, check everything + # stays the same + e.concretize() + + assert all('mpich2' in spec for _, spec in e.concretized_specs()) + assert all('mpich' not in spec for _, spec in e.concretized_specs()) + + +def test_cant_install_single_spec_when_concretizing_together(): + e = ev.create('coconcretization') + e.concretization = 'together' + + with pytest.raises(ev.SpackEnvironmentError, match=r'cannot install'): + e.install('zlib') + + +def test_duplicate_packages_raise_when_concretizing_together(): + e = ev.create('coconcretization') + e.concretization = 'together' + + e.add('mpileaks+opt') + e.add('mpileaks~opt') + e.add('mpich') + + with pytest.raises(ev.SpackEnvironmentError, match=r'cannot contain more'): + e.concretize()