Change environment modifications to escape with double quotes (#36789)

This commit changes the environment modifications class to escape
strings with double quotes instead of single quotes.

Single quotes prevent the expansion of enviornment variables that are
nested within environment variable definitions.

lib/spack/spack/test/util/environment.py

@@ -151,3 +151,21 @@ def test_reverse_environment_modifications(working_env):
 
     start_env.pop("UNSET")
     assert os.environ == start_env
+
+
+def test_escape_double_quotes_in_shell_modifications():
+    to_validate = envutil.EnvironmentModifications()
+
+    to_validate.set("VAR", "$PATH")
+    to_validate.append_path("VAR", "$ANOTHER_PATH")
+
+    to_validate.set("QUOTED_VAR", '"MY_VAL"')
+
+    cmds = to_validate.shell_modifications()
+
+    if sys.platform != "win32":
+        assert 'export VAR="$PATH:$ANOTHER_PATH"' in cmds
+        assert r'export QUOTED_VAR="\"MY_VAL\""' in cmds
+    else:
+        assert "export VAR=$PATH;$ANOTHER_PATH" in cmds
+        assert r'export QUOTED_VAR="MY_VAL"' in cmds

lib/spack/spack/util/environment.py

@@ -12,7 +12,6 @@
 import pickle
 import platform
 import re
-import shlex
 import socket
 import sys
 from typing import Any, Callable, Dict, List, MutableMapping, Optional, Tuple, Union
@@ -64,6 +63,26 @@
 ModificationList = List[Union["NameModifier", "NameValueModifier"]]
 
 
+_find_unsafe = re.compile(r"[^\w@%+=:,./-]", re.ASCII).search
+
+
+def double_quote_escape(s):
+    """Return a shell-escaped version of the string *s*.
+
+    This is similar to how shlex.quote works, but it escapes with double quotes
+    instead of single quotes, to allow environment variable expansion within
+    quoted strings.
+    """
+    if not s:
+        return '""'
+    if _find_unsafe(s) is None:
+        return s
+
+    # use double quotes, and escape double quotes in the string
+    # the string $"b is then quoted as "$\"b"
+    return '"' + s.replace('"', r"\"") + '"'
+
+
 def is_system_path(path: Path) -> bool:
     """Returns True if the argument is a system path, False otherwise."""
     return bool(path) and (os.path.normpath(path) in SYSTEM_DIRS)
@@ -135,7 +154,7 @@ def _env_var_to_source_line(var: str, val: str) -> str:
             fname=BASH_FUNCTION_FINDER.sub(r"\1", var), decl=val
         )
     else:
-        source_line = f"{var}={shlex.quote(val)}; export {var}"
+        source_line = f"{var}={double_quote_escape(val)}; export {var}"
     return source_line
 
 
@@ -649,7 +668,9 @@ def shell_modifications(
                     cmds += _SHELL_UNSET_STRINGS[shell].format(name)
                 else:
                     if sys.platform != "win32":
-                        cmd = _SHELL_SET_STRINGS[shell].format(name, shlex.quote(new_env[name]))
+                        cmd = _SHELL_SET_STRINGS[shell].format(
+                            name, double_quote_escape(new_env[name])
+                        )
                     else:
                         cmd = _SHELL_SET_STRINGS[shell].format(name, new_env[name])
                     cmds += cmd