adding spack -c to set one off config arguments (#22251)

This pull request will add the ability for a user to add a configuration argument on the fly, on the command line, e.g.,:

```bash
$ spack -c config:install_tree:root:/path/to/config.yaml -c packages:all:compiler:[gcc] list --help
```
The above command doesn't do anything (I'm just getting help for list) but you can imagine having another root of packages, and updating it on the fly for a command (something I'd like to do in the near future!)

I've moved the logic for config_add that used to be in spack/cmd/config.py into spack/config.py proper, and now both the main.py (where spack commands live) and spack/cmd/config.py use these functions. I only needed spack config add, so I didn't move the others. We can move the others if there are also needed in multiple places.

lib/spack/spack/cmd/config.py

@@ -200,71 +200,11 @@ def config_add(args):
 
     scope, section = _get_scope_and_section(args)
 
-    # Updates from file
     if args.file:
-        # Get file as config dict
+        spack.config.add_from_file(args.file, scope=scope)
-        data = spack.config.read_config_file(args.file)
-        if any(k in data for k in spack.schema.env.keys):
-            data = ev.config_dict(data)
-
-        # update all sections from config dict
-        # We have to iterate on keys to keep overrides from the file
-        for section in data.keys():
-            if section in spack.config.section_schemas.keys():
-                # Special handling for compiler scope difference
-                # Has to be handled after we choose a section
-                if scope is None:
-                    scope = spack.config.default_modify_scope(section)
-
-                value = data[section]
-                existing = spack.config.get(section, scope=scope)
-                new = spack.config.merge_yaml(existing, value)
-
-                spack.config.set(section, new, scope)
 
     if args.path:
-        components = spack.config.process_config_path(args.path)
+        spack.config.add(args.path, scope=scope)
-
-        has_existing_value = True
-        path = ''
-        override = False
-        for idx, name in enumerate(components[:-1]):
-            # First handle double colons in constructing path
-            colon = '::' if override else ':' if path else ''
-            path += colon + name
-            if getattr(name, 'override', False):
-                override = True
-            else:
-                override = False
-
-            # Test whether there is an existing value at this level
-            existing = spack.config.get(path, scope=scope)
-
-            if existing is None:
-                has_existing_value = False
-                # We've nested further than existing config, so we need the
-                # type information for validation to know how to handle bare
-                # values appended to lists.
-                existing = spack.config.get_valid_type(path)
-
-                # construct value from this point down
-                value = syaml.load_config(components[-1])
-                for component in reversed(components[idx + 1:-1]):
-                    value = {component: value}
-                break
-
-        if has_existing_value:
-            path, _, value = args.path.rpartition(':')
-            value = syaml.load_config(value)
-            existing = spack.config.get(path, scope=scope)
-
-        # append values to lists
-        if isinstance(existing, list) and not isinstance(value, list):
-            value = [value]
-
-        # merge value into existing
-        new = spack.config.merge_yaml(existing, value)
-        spack.config.set(path, new, scope)
 
 
 def config_remove(args):

lib/spack/spack/config.py

@@ -806,6 +806,81 @@ def _config():
 config = llnl.util.lang.Singleton(_config)
 
 
+def add_from_file(filename, scope=None):
+    """Add updates to a config from a filename
+    """
+    import spack.environment as ev
+
+    # Get file as config dict
+    data = read_config_file(filename)
+    if any(k in data for k in spack.schema.env.keys):
+        data = ev.config_dict(data)
+
+    # update all sections from config dict
+    # We have to iterate on keys to keep overrides from the file
+    for section in data.keys():
+        if section in section_schemas.keys():
+            # Special handling for compiler scope difference
+            # Has to be handled after we choose a section
+            if scope is None:
+                scope = default_modify_scope(section)
+
+            value = data[section]
+            existing = get(section, scope=scope)
+            new = merge_yaml(existing, value)
+
+            # We cannot call config.set directly (set is a type)
+            config.set(section, new, scope)
+
+
+def add(fullpath, scope=None):
+    """Add the given configuration to the specified config scope.
+    Add accepts a path. If you want to add from a filename, use add_from_file"""
+
+    components = process_config_path(fullpath)
+
+    has_existing_value = True
+    path = ''
+    override = False
+    for idx, name in enumerate(components[:-1]):
+        # First handle double colons in constructing path
+        colon = '::' if override else ':' if path else ''
+        path += colon + name
+        if getattr(name, 'override', False):
+            override = True
+        else:
+            override = False
+
+        # Test whether there is an existing value at this level
+        existing = get(path, scope=scope)
+
+        if existing is None:
+            has_existing_value = False
+            # We've nested further than existing config, so we need the
+            # type information for validation to know how to handle bare
+            # values appended to lists.
+            existing = get_valid_type(path)
+
+            # construct value from this point down
+            value = syaml.load_config(components[-1])
+            for component in reversed(components[idx + 1:-1]):
+                value = {component: value}
+            break
+
+    if has_existing_value:
+        path, _, value = fullpath.rpartition(':')
+        value = syaml.load_config(value)
+        existing = get(path, scope=scope)
+
+    # append values to lists
+    if isinstance(existing, list) and not isinstance(value, list):
+        value = [value]
+
+    # merge value into existing
+    new = merge_yaml(existing, value)
+    config.set(path, new, scope)
+
+
 def get(path, default=None, scope=None):
     """Module-level wrapper for ``Configuration.get()``."""
     return config.get(path, default, scope)

lib/spack/spack/main.py

@@ -40,7 +40,6 @@
 import spack.util.executable as exe
 from spack.error import SpackError
 
-
 #: names of profile statistics
 stat_names = pstats.Stats.sort_arg_dict_default
 
@@ -358,6 +357,9 @@ def make_argument_parser(**kwargs):
         '--color', action='store', default='auto',
         choices=('always', 'never', 'auto'),
         help="when to colorize output (default: auto)")
+    parser.add_argument(
+        '-c', '--config', default=None, action="append", dest="config_vars",
+        help="add one or more custom, one off config settings.")
     parser.add_argument(
         '-C', '--config-scope', dest='config_scopes', action='append',
         metavar='DIR', help="add a custom configuration scope")
@@ -463,6 +465,10 @@ def setup_main_options(args):
         tty.warn("You asked for --insecure. Will NOT check SSL certificates.")
         spack.config.set('config:verify_ssl', False, scope='command_line')
 
+    # Use the spack config command to handle parsing the config strings
+    for config_var in (args.config_vars or []):
+        spack.config.add(path=config_var, scope="command_line")
+
     # when to use color (takes always, auto, or never)
     color.set_color_when(args.color)
 

lib/spack/spack/test/cmd/config.py

@@ -87,6 +87,7 @@ def test_get_config_scope_merged(mock_low_high_config):
 
 def test_config_edit():
     """Ensure `spack config edit` edits the right paths."""
+
     dms = spack.config.default_modify_scope('compilers')
     dms_path = spack.config.config.scopes[dms].path
     user_path = spack.config.config.scopes['user'].path
@@ -204,20 +205,27 @@ def test_config_add_override_leaf(mutable_empty_config):
 
 
 def test_config_add_update_dict(mutable_empty_config):
-    config('add', 'packages:all:compiler:[gcc]')
+    config('add', 'packages:all:version:[1.0.0]')
-    config('add', 'packages:all:version:1.0.0')
     output = config('get', 'packages')
 
-    expected = """packages:
+    expected = 'packages:\n  all:\n    version: [1.0.0]\n'
-  all:
-    compiler: [gcc]
-    version:
-    - 1.0.0
-"""
-
     assert output == expected
 
 
+def test_config_with_c_argument(mutable_empty_config):
+
+    # I don't know how to add a spack argument to a Spack Command, so we test this way
+    config_file = 'config:install_root:root:/path/to/config.yaml'
+    parser = spack.main.make_argument_parser()
+    args = parser.parse_args(['-c', config_file])
+    assert config_file in args.config_vars
+
+    # Add the path to the config
+    config("add", args.config_vars[0], scope='command_line')
+    output = config("get", 'config')
+    assert "config:\n  install_root:\n  - root: /path/to/config.yaml" in output
+
+
 def test_config_add_ordered_dict(mutable_empty_config):
     config('add', 'mirrors:first:/path/to/first')
     config('add', 'mirrors:second:/path/to/second')

lib/spack/spack/test/config.py

@@ -258,6 +258,37 @@ def test_write_to_same_priority_file(mock_low_high_config, compiler_specs):
 repos_low = {'repos': ["/some/path"]}
 repos_high = {'repos': ["/some/other/path"]}
 
+# Test setting config values via path in filename
+
+
+def test_add_config_path():
+
+    # Try setting a new install tree root
+    path = "config:install_tree:root:/path/to/config.yaml"
+    spack.config.add(path, scope="command_line")
+    set_value = spack.config.get('config')['install_tree']['root']
+    assert set_value == '/path/to/config.yaml'
+
+    # Now a package:all setting
+    path = "packages:all:compiler:[gcc]"
+    spack.config.add(path, scope="command_line")
+    compilers = spack.config.get('packages')['all']['compiler']
+    assert "gcc" in compilers
+
+
+def test_add_config_filename(mock_low_high_config, tmpdir):
+
+    config_yaml = tmpdir.join('config-filename.yaml')
+    config_yaml.ensure()
+    with config_yaml.open('w') as f:
+        syaml.dump_config(config_low, f)
+
+    spack.config.add_from_file(str(config_yaml), scope="low")
+    assert "build_stage" in spack.config.get('config')
+    build_stages = spack.config.get('config')['build_stage']
+    for stage in config_low['config']['build_stage']:
+        assert stage in build_stages
+
 
 # repos
 def test_write_list_in_memory(mock_low_high_config):

share/spack/spack-completion.bash

@@ -331,7 +331,7 @@ _spacktivate() {
 _spack() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
+        SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
     else
         SPACK_COMPREPLY="activate add arch blame build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
     fi