spack deconcretize command (#38803)

We have two ways to concretize now:
* `spack concretize` concretizes only the root specs that are not concrete in the environment.
* `spack concretize -f` eliminates all cached concretization data and reconcretizes the *entire* environment.

This PR adds `spack deconcretize`, which eliminates cached concretization data for a spec.  This allows
users greater control over what is preserved from their `spack.lock` file and what is reused when not
using `spack concretize -f`.  If you want to update a spec installed in your environment, you can call
`spack deconcretize` on it, and that spec and any relevant dependents will be removed from the lock file.

`spack concretize` has two options:
* `--root`: limits deconcretized specs to *specific* roots in the environment. You can use this to
  deconcretize exactly one root in a `unify: false` environment.  i.e., if `foo` root is a dependent
  of `bar`, both roots, `spack deconcretize bar` will *not* deconcretize `foo`.
* `--all`: deconcretize *all* specs that match the input spec. By default `spack deconcretize`
  will complain about multiple matches, like `spack uninstall`.

lib/spack/spack/cmd/common/confirmation.py

@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import sys
+from typing import List
+
+import llnl.util.tty as tty
+
+import spack.cmd
+
+display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4}
+
+
+def confirm_action(specs: List[spack.spec.Spec], participle: str, noun: str):
+    """Display the list of specs to be acted on and ask for confirmation.
+
+    Args:
+        specs: specs to be removed
+        participle: action expressed as a participle, e.g. "uninstalled"
+        noun: action expressed as a noun, e.g. "uninstallation"
+    """
+    tty.msg(f"The following {len(specs)} packages will be {participle}:\n")
+    spack.cmd.display_specs(specs, **display_args)
+    print("")
+    answer = tty.get_yes_or_no("Do you want to proceed?", default=False)
+    if not answer:
+        tty.msg(f"Aborting {noun}")
+        sys.exit(0)

lib/spack/spack/cmd/deconcretize.py

@@ -0,0 +1,103 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import argparse
+import sys
+from typing import List
+
+import llnl.util.tty as tty
+
+import spack.cmd
+import spack.cmd.common.arguments as arguments
+import spack.cmd.common.confirmation as confirmation
+import spack.environment as ev
+import spack.spec
+
+description = "remove specs from the concretized lockfile of an environment"
+section = "environments"
+level = "long"
+
+# Arguments for display_specs when we find ambiguity
+display_args = {"long": True, "show_flags": False, "variants": False, "indent": 4}
+
+
+def setup_parser(subparser):
+    subparser.add_argument(
+        "--root", action="store_true", help="deconcretize only specific environment roots"
+    )
+    arguments.add_common_arguments(subparser, ["yes_to_all", "specs"])
+    subparser.add_argument(
+        "-a",
+        "--all",
+        action="store_true",
+        dest="all",
+        help="deconcretize ALL specs that match each supplied spec",
+    )
+
+
+def get_deconcretize_list(
+    args: argparse.Namespace, specs: List[spack.spec.Spec], env: ev.Environment
+) -> List[spack.spec.Spec]:
+    """
+    Get list of environment roots to deconcretize
+    """
+    env_specs = [s for _, s in env.concretized_specs()]
+    to_deconcretize = []
+    errors = []
+
+    for s in specs:
+        if args.root:
+            # find all roots matching given spec
+            to_deconc = [e for e in env_specs if e.satisfies(s)]
+        else:
+            # find all roots matching or depending on a matching spec
+            to_deconc = [e for e in env_specs if any(d.satisfies(s) for d in e.traverse())]
+
+        if len(to_deconc) < 1:
+            tty.warn(f"No matching specs to deconcretize for {s}")
+
+        elif len(to_deconc) > 1 and not args.all:
+            errors.append((s, to_deconc))
+
+        to_deconcretize.extend(to_deconc)
+
+    if errors:
+        for spec, matching in errors:
+            tty.error(f"{spec} matches multiple concrete specs:")
+            sys.stderr.write("\n")
+            spack.cmd.display_specs(matching, output=sys.stderr, **display_args)
+            sys.stderr.write("\n")
+            sys.stderr.flush()
+        tty.die("Use '--all' to deconcretize all matching specs, or be more specific")
+
+    return to_deconcretize
+
+
+def deconcretize_specs(args, specs):
+    env = spack.cmd.require_active_env(cmd_name="deconcretize")
+
+    if args.specs:
+        deconcretize_list = get_deconcretize_list(args, specs, env)
+    else:
+        deconcretize_list = [s for _, s in env.concretized_specs()]
+
+    if not args.yes_to_all:
+        confirmation.confirm_action(deconcretize_list, "deconcretized", "deconcretization")
+
+    with env.write_transaction():
+        for spec in deconcretize_list:
+            env.deconcretize(spec)
+        env.write()
+
+
+def deconcretize(parser, args):
+    if not args.specs and not args.all:
+        tty.die(
+            "deconcretize requires at least one spec argument.",
+            " Use `spack deconcretize --all` to deconcretize ALL specs.",
+        )
+
+    specs = spack.cmd.parse_specs(args.specs) if args.specs else [any]
+    deconcretize_specs(args, specs)

lib/spack/spack/cmd/gc.py

@@ -6,6 +6,7 @@
 import llnl.util.tty as tty
 
 import spack.cmd.common.arguments
+import spack.cmd.common.confirmation
 import spack.cmd.uninstall
 import spack.environment as ev
 import spack.store
@@ -41,6 +42,6 @@ def gc(parser, args):
         return
 
     if not args.yes_to_all:
-        spack.cmd.uninstall.confirm_removal(specs)
+        spack.cmd.common.confirmation.confirm_action(specs, "uninstalled", "uninstallation")
 
     spack.cmd.uninstall.do_uninstall(specs, force=False)

lib/spack/spack/cmd/uninstall.py

@@ -11,10 +11,9 @@
 
 import spack.cmd
 import spack.cmd.common.arguments as arguments
+import spack.cmd.common.confirmation as confirmation
 import spack.environment as ev
-import spack.error
 import spack.package_base
-import spack.repo
 import spack.spec
 import spack.store
 import spack.traverse as traverse
@@ -278,7 +277,7 @@ def uninstall_specs(args, specs):
         return
 
     if not args.yes_to_all:
-        confirm_removal(uninstall_list)
+        confirmation.confirm_action(uninstall_list, "uninstalled", "uninstallation")
 
     # Uninstall everything on the list
     do_uninstall(uninstall_list, args.force)
@@ -292,21 +291,6 @@ def uninstall_specs(args, specs):
         env.regenerate_views()
 
 
-def confirm_removal(specs: List[spack.spec.Spec]):
-    """Display the list of specs to be removed and ask for confirmation.
-
-    Args:
-        specs: specs to be removed
-    """
-    tty.msg("The following {} packages will be uninstalled:\n".format(len(specs)))
-    spack.cmd.display_specs(specs, **display_args)
-    print("")
-    answer = tty.get_yes_or_no("Do you want to proceed?", default=False)
-    if not answer:
-        tty.msg("Aborting uninstallation")
-        sys.exit(0)
-
-
 def uninstall(parser, args):
     if not args.specs and not args.all:
         tty.die(

lib/spack/spack/environment/environment.py

@@ -1358,7 +1358,7 @@ def concretize(self, force=False, tests=False):
 
         # Remove concrete specs that no longer correlate to a user spec
         for spec in set(self.concretized_user_specs) - set(self.user_specs):
-            self.deconcretize(spec)
+            self.deconcretize(spec, concrete=False)
 
         # Pick the right concretization strategy
         if self.unify == "when_possible":
@@ -1373,14 +1373,35 @@ def concretize(self, force=False, tests=False):
         msg = "concretization strategy not implemented [{0}]"
         raise SpackEnvironmentError(msg.format(self.unify))
 
-    def deconcretize(self, spec):
+    def deconcretize(self, spec: spack.spec.Spec, concrete: bool = True):
+        """
+        Remove specified spec from environment concretization
+
+        Arguments:
+            spec: Spec to deconcretize. This must be a root of the environment
+            concrete: If True, find all instances of spec as concrete in the environemnt.
+                If False, find a single instance of the abstract spec as root of the environment.
+        """
         # spec has to be a root of the environment
+        if concrete:
+            dag_hash = spec.dag_hash()
+
+            pairs = zip(self.concretized_user_specs, self.concretized_order)
+            filtered = [(spec, h) for spec, h in pairs if h != dag_hash]
+            # Cannot use zip and unpack two values; it fails if filtered is empty
+            self.concretized_user_specs = [s for s, _ in filtered]
+            self.concretized_order = [h for _, h in filtered]
+        else:
             index = self.concretized_user_specs.index(spec)
             dag_hash = self.concretized_order.pop(index)
+
             del self.concretized_user_specs[index]
 
         # If this was the only user spec that concretized to this concrete spec, remove it
         if dag_hash not in self.concretized_order:
+            # if we deconcretized a dependency that doesn't correspond to a root, it
+            # won't be here.
+            if dag_hash in self.specs_by_hash:
                 del self.specs_by_hash[dag_hash]
 
     def _get_specs_to_concretize(

lib/spack/spack/test/cmd/deconcretize.py

@@ -0,0 +1,78 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import pytest
+
+import spack.environment as ev
+from spack.main import SpackCommand, SpackCommandError
+
+deconcretize = SpackCommand("deconcretize")
+
+
+@pytest.fixture(scope="function")
+def test_env(mutable_mock_env_path, config, mock_packages):
+    ev.create("test")
+    with ev.read("test") as e:
+        e.add("a@2.0 foobar=bar ^b@1.0")
+        e.add("a@1.0 foobar=bar ^b@0.9")
+        e.concretize()
+        e.write()
+
+
+def test_deconcretize_dep(test_env):
+    with ev.read("test") as e:
+        deconcretize("-y", "b@1.0")
+        specs = [s for s, _ in e.concretized_specs()]
+
+    assert len(specs) == 1
+    assert specs[0].satisfies("a@1.0")
+
+
+def test_deconcretize_all_dep(test_env):
+    with ev.read("test") as e:
+        with pytest.raises(SpackCommandError):
+            deconcretize("-y", "b")
+        deconcretize("-y", "--all", "b")
+        specs = [s for s, _ in e.concretized_specs()]
+
+    assert len(specs) == 0
+
+
+def test_deconcretize_root(test_env):
+    with ev.read("test") as e:
+        output = deconcretize("-y", "--root", "b@1.0")
+        assert "No matching specs to deconcretize" in output
+        assert len(e.concretized_order) == 2
+
+        deconcretize("-y", "--root", "a@2.0")
+        specs = [s for s, _ in e.concretized_specs()]
+
+    assert len(specs) == 1
+    assert specs[0].satisfies("a@1.0")
+
+
+def test_deconcretize_all_root(test_env):
+    with ev.read("test") as e:
+        with pytest.raises(SpackCommandError):
+            deconcretize("-y", "--root", "a")
+
+        output = deconcretize("-y", "--root", "--all", "b")
+        assert "No matching specs to deconcretize" in output
+        assert len(e.concretized_order) == 2
+
+        deconcretize("-y", "--root", "--all", "a")
+        specs = [s for s, _ in e.concretized_specs()]
+
+    assert len(specs) == 0
+
+
+def test_deconcretize_all(test_env):
+    with ev.read("test") as e:
+        with pytest.raises(SpackCommandError):
+            deconcretize()
+        deconcretize("-y", "--all")
+        specs = [s for s, _ in e.concretized_specs()]
+
+    assert len(specs) == 0

share/spack/spack-completion.bash

@@ -401,7 +401,7 @@ _spack() {
     then
         SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace --backtrace -V --version --print-shell-vars"
     else
-        SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
+        SPACK_COMPREPLY="add arch audit blame bootstrap build-env buildcache cd change checksum ci clean clone commands compiler compilers concretize concretise config containerize containerise create debug deconcretize dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers make-installer mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
     fi
 }
 
@@ -937,6 +937,15 @@ _spack_debug_report() {
     SPACK_COMPREPLY="-h --help"
 }
 
+_spack_deconcretize() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help --root -y --yes-to-all -a --all"
+    else
+        _all_packages
+    fi
+}
+
 _spack_dependencies() {
     if $list_options
     then

share/spack/spack-completion.fish

@@ -371,6 +371,7 @@ complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a containerize -d '
 complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a containerise -d 'creates recipes to build images for different container runtimes'
 complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a create -d 'create a new package file'
 complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a debug -d 'debugging commands for troubleshooting Spack'
+complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a deconcretize -d 'remove specs from the concretized lockfile of an environment'
 complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dependencies -d 'show dependencies of a package'
 complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a dependents -d 'show packages that depend on another'
 complete -c spack -n '__fish_spack_using_command_pos 0 ' -f -a deprecate -d 'replace one package with another via symlinks'
@@ -1290,6 +1291,18 @@ set -g __fish_spack_optspecs_spack_debug_report h/help
 complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command debug report' -s h -l help -d 'show this help message and exit'
 
+# spack deconcretize
+set -g __fish_spack_optspecs_spack_deconcretize h/help root y/yes-to-all a/all
+complete -c spack -n '__fish_spack_using_command_pos_remainder 0 deconcretize' -f -k -a '(__fish_spack_specs)'
+complete -c spack -n '__fish_spack_using_command deconcretize' -s h -l help -f -a help
+complete -c spack -n '__fish_spack_using_command deconcretize' -s h -l help -d 'show this help message and exit'
+complete -c spack -n '__fish_spack_using_command deconcretize' -l root -f -a root
+complete -c spack -n '__fish_spack_using_command deconcretize' -l root -d 'deconcretize only specific environment roots'
+complete -c spack -n '__fish_spack_using_command deconcretize' -s y -l yes-to-all -f -a yes_to_all
+complete -c spack -n '__fish_spack_using_command deconcretize' -s y -l yes-to-all -d 'assume "yes" is the answer to every confirmation request'
+complete -c spack -n '__fish_spack_using_command deconcretize' -s a -l all -f -a all
+complete -c spack -n '__fish_spack_using_command deconcretize' -s a -l all -d 'deconcretize ALL specs that match each supplied spec'
+
 # spack dependencies
 set -g __fish_spack_optspecs_spack_dependencies h/help i/installed t/transitive deptype= V/no-expand-virtuals
 complete -c spack -n '__fish_spack_using_command_pos_remainder 0 dependencies' -f -k -a '(__fish_spack_specs)'