From 09e9bb5c3d5e568f13f50e11c7ae357bcad71d9d Mon Sep 17 00:00:00 2001 From: Greg Becker Date: Fri, 10 Nov 2023 14:55:35 -0800 Subject: [PATCH] `spack deconcretize` command (#38803) We have two ways to concretize now: * `spack concretize` concretizes only the root specs that are not concrete in the environment. * `spack concretize -f` eliminates all cached concretization data and reconcretizes the *entire* environment. This PR adds `spack deconcretize`, which eliminates cached concretization data for a spec. This allows users greater control over what is preserved from their `spack.lock` file and what is reused when not using `spack concretize -f`. If you want to update a spec installed in your environment, you can call `spack deconcretize` on it, and that spec and any relevant dependents will be removed from the lock file. `spack concretize` has two options: * `--root`: limits deconcretized specs to *specific* roots in the environment. You can use this to deconcretize exactly one root in a `unify: false` environment. i.e., if `foo` root is a dependent of `bar`, both roots, `spack deconcretize bar` will *not* deconcretize `foo`. * `--all`: deconcretize *all* specs that match the input spec. By default `spack deconcretize` will complain about multiple matches, like `spack uninstall`. --- lib/spack/spack/cmd/common/confirmation.py | 30 ++++++ lib/spack/spack/cmd/deconcretize.py | 103 +++++++++++++++++++++ lib/spack/spack/cmd/gc.py | 3 +- lib/spack/spack/cmd/uninstall.py | 20 +--- lib/spack/spack/environment/environment.py | 33 +++++-- lib/spack/spack/test/cmd/deconcretize.py | 78 ++++++++++++++++ share/spack/spack-completion.bash | 11 ++- share/spack/spack-completion.fish | 13 +++ 8 files changed, 265 insertions(+), 26 deletions(-) create mode 100644 lib/spack/spack/cmd/common/confirmation.py create mode 100644 lib/spack/spack/cmd/deconcretize.py create mode 100644 lib/spack/spack/test/cmd/deconcretize.py diff --git a/lib/spack/spack/cmd/common/confirmation.py b/lib/spack/spack/cmd/common/confirmation.py new file mode 100644 index 0000000000..8a5cd2592b --- /dev/null +++ b/lib/spack/spack/cmd/common/confirmation.py @@ -0,0 +1,30 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import sys +from typing import List + +import llnl.util.tty as tty + +import spack.cmd + +display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} + + +def confirm_action(specs: List[spack.spec.Spec], participle: str, noun: str): + """Display the list of specs to be acted on and ask for confirmation. + + Args: + specs: specs to be removed + participle: action expressed as a participle, e.g. "uninstalled" + noun: action expressed as a noun, e.g. "uninstallation" + """ + tty.msg(f"The following {len(specs)} packages will be {participle}:\n") + spack.cmd.display_specs(specs, **display_args) + print("") + answer = tty.get_yes_or_no("Do you want to proceed?", default=False) + if not answer: + tty.msg(f"Aborting {noun}") + sys.exit(0) diff --git a/lib/spack/spack/cmd/deconcretize.py b/lib/spack/spack/cmd/deconcretize.py new file mode 100644 index 0000000000..dbcf72ea8b --- /dev/null +++ b/lib/spack/spack/cmd/deconcretize.py @@ -0,0 +1,103 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import argparse +import sys +from typing import List + +import llnl.util.tty as tty + +import spack.cmd +import spack.cmd.common.arguments as arguments +import spack.cmd.common.confirmation as confirmation +import spack.environment as ev +import spack.spec + +description = "remove specs from the concretized lockfile of an environment" +section = "environments" +level = "long" + +# Arguments for display_specs when we find ambiguity +display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4} + + +def setup_parser(subparser): + subparser.add_argument( + "--root", action="store_true", help="deconcretize only specific environment roots" + ) + arguments.add_common_arguments(subparser, ["yes_to_all", "specs"]) + subparser.add_argument( + "-a", + "--all", + action="store_true", + dest="all", + help="deconcretize ALL specs that match each supplied spec", + ) + + +def get_deconcretize_list( + args: argparse.Namespace, specs: List[spack.spec.Spec], env: ev.Environment +) -> List[spack.spec.Spec]: + """ + Get list of environment roots to deconcretize + """ + env_specs = [s for _, s in env.concretized_specs()] + to_deconcretize = [] + errors = [] + + for s in specs: + if args.root: + # find all roots matching given spec + to_deconc = [e for e in env_specs if e.satisfies(s)] + else: + # find all roots matching or depending on a matching spec + to_deconc = [e for e in env_specs if any(d.satisfies(s) for d in e.traverse())] + + if len(to_deconc) < 1: + tty.warn(f"No matching specs to deconcretize for {s}") + + elif len(to_deconc) > 1 and not args.all: + errors.append((s, to_deconc)) + + to_deconcretize.extend(to_deconc) + + if errors: + for spec, matching in errors: + tty.error(f"{spec} matches multiple concrete specs:") + sys.stderr.write("\n") + spack.cmd.display_specs(matching, output=sys.stderr, **display_args) + sys.stderr.write("\n") + sys.stderr.flush() + tty.die("Use '--all' to deconcretize all matching specs, or be more specific") + + return to_deconcretize + + +def deconcretize_specs(args, specs): + env = spack.cmd.require_active_env(cmd_name="deconcretize") + + if args.specs: + deconcretize_list = get_deconcretize_list(args, specs, env) + else: + deconcretize_list = [s for _, s in env.concretized_specs()] + + if not args.yes_to_all: + confirmation.confirm_action(deconcretize_list, "deconcretized", "deconcretization") + + with env.write_transaction(): + for spec in deconcretize_list: + env.deconcretize(spec) + env.write() + + +def deconcretize(parser, args): + if not args.specs and not args.all: + tty.die( + "deconcretize requires at least one spec argument.", + " Use `spack deconcretize --all` to deconcretize ALL specs.", + ) + + specs = spack.cmd.parse_specs(args.specs) if args.specs else [any] + deconcretize_specs(args, specs) diff --git a/lib/spack/spack/cmd/gc.py b/lib/spack/spack/cmd/gc.py index e4da6a103d..9918bf7479 100644 --- a/lib/spack/spack/cmd/gc.py +++ b/lib/spack/spack/cmd/gc.py @@ -6,6 +6,7 @@ import llnl.util.tty as tty import spack.cmd.common.arguments +import spack.cmd.common.confirmation import spack.cmd.uninstall import spack.environment as ev import spack.store @@ -41,6 +42,6 @@ def gc(parser, args): return if not args.yes_to_all: - spack.cmd.uninstall.confirm_removal(specs) + spack.cmd.common.confirmation.confirm_action(specs, "uninstalled", "uninstallation") spack.cmd.uninstall.do_uninstall(specs, force=False) diff --git a/lib/spack/spack/cmd/uninstall.py b/lib/spack/spack/cmd/uninstall.py index bc6a71cef1..3288404151 100644 --- a/lib/spack/spack/cmd/uninstall.py +++ b/lib/spack/spack/cmd/uninstall.py @@ -11,10 +11,9 @@ import spack.cmd import spack.cmd.common.arguments as arguments +import spack.cmd.common.confirmation as confirmation import spack.environment as ev -import spack.error import spack.package_base -import spack.repo import spack.spec import spack.store import spack.traverse as traverse @@ -278,7 +277,7 @@ def uninstall_specs(args, specs): return if not args.yes_to_all: - confirm_removal(uninstall_list) + confirmation.confirm_action(uninstall_list, "uninstalled", "uninstallation") # Uninstall everything on the list do_uninstall(uninstall_list, args.force) @@ -292,21 +291,6 @@ def uninstall_specs(args, specs): env.regenerate_views() -def confirm_removal(specs: List[spack.spec.Spec]): - """Display the list of specs to be removed and ask for confirmation. - - Args: - specs: specs to be removed - """ - tty.msg("The following {} packages will be uninstalled:\n".format(len(specs))) - spack.cmd.display_specs(specs, **display_args) - print("") - answer = tty.get_yes_or_no("Do you want to proceed?", default=False) - if not answer: - tty.msg("Aborting uninstallation") - sys.exit(0) - - def uninstall(parser, args): if not args.specs and not args.all: tty.die( diff --git a/lib/spack/spack/environment/environment.py b/lib/spack/spack/environment/environment.py index cf6dffcb0d..5d6273506e 100644 --- a/lib/spack/spack/environment/environment.py +++ b/lib/spack/spack/environment/environment.py @@ -1358,7 +1358,7 @@ def concretize(self, force=False, tests=False): # Remove concrete specs that no longer correlate to a user spec for spec in set(self.concretized_user_specs) - set(self.user_specs): - self.deconcretize(spec) + self.deconcretize(spec, concrete=False) # Pick the right concretization strategy if self.unify == "when_possible": @@ -1373,15 +1373,36 @@ def concretize(self, force=False, tests=False): msg = "concretization strategy not implemented [{0}]" raise SpackEnvironmentError(msg.format(self.unify)) - def deconcretize(self, spec): + def deconcretize(self, spec: spack.spec.Spec, concrete: bool = True): + """ + Remove specified spec from environment concretization + + Arguments: + spec: Spec to deconcretize. This must be a root of the environment + concrete: If True, find all instances of spec as concrete in the environemnt. + If False, find a single instance of the abstract spec as root of the environment. + """ # spec has to be a root of the environment - index = self.concretized_user_specs.index(spec) - dag_hash = self.concretized_order.pop(index) - del self.concretized_user_specs[index] + if concrete: + dag_hash = spec.dag_hash() + + pairs = zip(self.concretized_user_specs, self.concretized_order) + filtered = [(spec, h) for spec, h in pairs if h != dag_hash] + # Cannot use zip and unpack two values; it fails if filtered is empty + self.concretized_user_specs = [s for s, _ in filtered] + self.concretized_order = [h for _, h in filtered] + else: + index = self.concretized_user_specs.index(spec) + dag_hash = self.concretized_order.pop(index) + + del self.concretized_user_specs[index] # If this was the only user spec that concretized to this concrete spec, remove it if dag_hash not in self.concretized_order: - del self.specs_by_hash[dag_hash] + # if we deconcretized a dependency that doesn't correspond to a root, it + # won't be here. + if dag_hash in self.specs_by_hash: + del self.specs_by_hash[dag_hash] def _get_specs_to_concretize( self, diff --git a/lib/spack/spack/test/cmd/deconcretize.py b/lib/spack/spack/test/cmd/deconcretize.py new file mode 100644 index 0000000000..30e39604bf --- /dev/null +++ b/lib/spack/spack/test/cmd/deconcretize.py @@ -0,0 +1,78 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import pytest + +import spack.environment as ev +from spack.main import SpackCommand, SpackCommandError + +deconcretize = SpackCommand("deconcretize") + + +@pytest.fixture(scope="function") +def test_env(mutable_mock_env_path, config, mock_packages): + ev.create("test") + with ev.read("test") as e: + e.add("a@2.0 foobar=bar ^b@1.0") + e.add("a@1.0 foobar=bar ^b@0.9") + e.concretize() + e.write() + + +def test_deconcretize_dep(test_env): + with ev.read("test") as e: + deconcretize("-y", "b@1.0") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 1 + assert specs[0].satisfies("a@1.0") + + +def test_deconcretize_all_dep(test_env): + with ev.read("test") as e: + with pytest.raises(SpackCommandError): + deconcretize("-y", "b") + deconcretize("-y", "--all", "b") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 0 + + +def test_deconcretize_root(test_env): + with ev.read("test") as e: + output = deconcretize("-y", "--root", "b@1.0") + assert "No matching specs to deconcretize" in output + assert len(e.concretized_order) == 2 + + deconcretize("-y", "--root", "a@2.0") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 1 + assert specs[0].satisfies("a@1.0") + + +def test_deconcretize_all_root(test_env): + with ev.read("test") as e: + with pytest.raises(SpackCommandError): + deconcretize("-y", "--root", "a") + + output = deconcretize("-y", "--root", "--all", "b") + assert "No matching specs to deconcretize" in output + assert len(e.concretized_order) == 2 + + deconcretize("-y", "--root", "--all", "a") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 0 + + +def test_deconcretize_all(test_env): + with ev.read("test") as e: + with pytest.raises(SpackCommandError): + deconcretize() + deconcretize("-y", "--all") + specs = [s for s, _ in e.concretized_specs()] + + assert len(specs) == 0 diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index e84fe10134..a54f7db414 100755 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -401,7 +401,7 @@ _spack() { then SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace --backtrace -V --version --print-shell-vars" else - SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" + SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" fi } @@ -937,6 +937,15 @@ _spack_debug_report() { SPACK_COMPREPLY="-h --help" } +_spack_deconcretize() { + if $list_options + then + SPACK_COMPREPLY="-h --help --root -y --yes-to-all -a --all" + else + _all_packages + fi +} + _spack_dependencies() { if $list_options then diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index d660c251af..1029fa6b45 100755 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -371,6 +371,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a containerize -d ' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a containerise -d 'creates recipes to build images for different container runtimes' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a create -d 'create a new package file' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a debug -d 'debugging commands for troubleshooting Spack' +complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a deconcretize -d 'remove specs from the concretized lockfile of an environment' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dependencies -d 'show dependencies of a package' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dependents -d 'show packages that depend on another' complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a deprecate -d 'replace one package with another via symlinks' @@ -1290,6 +1291,18 @@ set -g __fish_spack_optspecs_spack_debug_report h/help complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -d 'show this help message and exit' +# spack deconcretize +set -g __fish_spack_optspecs_spack_deconcretize h/help root y/yes-to-all a/all +complete -c spack -n '__fish_spack_using_command_pos_remainder 0 deconcretize' -f -k -a '(__fish_spack_specs)' +complete -c spack -n '__fish_spack_using_command deconcretize' -s h -l help -f -a help +complete -c spack -n '__fish_spack_using_command deconcretize' -s h -l help -d 'show this help message and exit' +complete -c spack -n '__fish_spack_using_command deconcretize' -l root -f -a root +complete -c spack -n '__fish_spack_using_command deconcretize' -l root -d 'deconcretize only specific environment roots' +complete -c spack -n '__fish_spack_using_command deconcretize' -s y -l yes-to-all -f -a yes_to_all +complete -c spack -n '__fish_spack_using_command deconcretize' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request' +complete -c spack -n '__fish_spack_using_command deconcretize' -s a -l all -f -a all +complete -c spack -n '__fish_spack_using_command deconcretize' -s a -l all -d 'deconcretize ALL specs that match each supplied spec' + # spack dependencies set -g __fish_spack_optspecs_spack_dependencies h/help i/installed t/transitive deptype= V/no-expand-virtuals complete -c spack -n '__fish_spack_using_command_pos_remainder 0 dependencies' -f -k -a '(__fish_spack_specs)'