adding spack -c to set one off config arguments (#22251)
This pull request will add the ability for a user to add a configuration argument on the fly, on the command line, e.g.,: ```bash $ spack -c config:install_tree:root:/path/to/config.yaml -c packages:all:compiler:[gcc] list --help ``` The above command doesn't do anything (I'm just getting help for list) but you can imagine having another root of packages, and updating it on the fly for a command (something I'd like to do in the near future!) I've moved the logic for config_add that used to be in spack/cmd/config.py into spack/config.py proper, and now both the main.py (where spack commands live) and spack/cmd/config.py use these functions. I only needed spack config add, so I didn't move the others. We can move the others if there are also needed in multiple places.
This commit is contained in:
Vanessasaurus
GitHub
parent
839af2bd70
commit
746081e933
6 changed files with 133 additions and 73 deletions
|
|
@ -200,71 +200,11 @@ def config_add(args):
|
|||
|
||||
scope, section = _get_scope_and_section(args)
|
||||
|
||||
# Updates from file
|
||||
if args.file:
|
||||
# Get file as config dict
|
||||
data = spack.config.read_config_file(args.file)
|
||||
if any(k in data for k in spack.schema.env.keys):
|
||||
data = ev.config_dict(data)
|
||||
|
||||
# update all sections from config dict
|
||||
# We have to iterate on keys to keep overrides from the file
|
||||
for section in data.keys():
|
||||
if section in spack.config.section_schemas.keys():
|
||||
# Special handling for compiler scope difference
|
||||
# Has to be handled after we choose a section
|
||||
if scope is None:
|
||||
scope = spack.config.default_modify_scope(section)
|
||||
|
||||
value = data[section]
|
||||
existing = spack.config.get(section, scope=scope)
|
||||
new = spack.config.merge_yaml(existing, value)
|
||||
|
||||
spack.config.set(section, new, scope)
|
||||
spack.config.add_from_file(args.file, scope=scope)
|
||||
|
||||
if args.path:
|
||||
components = spack.config.process_config_path(args.path)
|
||||
|
||||
has_existing_value = True
|
||||
path = ''
|
||||
override = False
|
||||
for idx, name in enumerate(components[:-1]):
|
||||
# First handle double colons in constructing path
|
||||
colon = '::' if override else ':' if path else ''
|
||||
path += colon + name
|
||||
if getattr(name, 'override', False):
|
||||
override = True
|
||||
else:
|
||||
override = False
|
||||
|
||||
# Test whether there is an existing value at this level
|
||||
existing = spack.config.get(path, scope=scope)
|
||||
|
||||
if existing is None:
|
||||
has_existing_value = False
|
||||
# We've nested further than existing config, so we need the
|
||||
# type information for validation to know how to handle bare
|
||||
# values appended to lists.
|
||||
existing = spack.config.get_valid_type(path)
|
||||
|
||||
# construct value from this point down
|
||||
value = syaml.load_config(components[-1])
|
||||
for component in reversed(components[idx + 1:-1]):
|
||||
value = {component: value}
|
||||
break
|
||||
|
||||
if has_existing_value:
|
||||
path, _, value = args.path.rpartition(':')
|
||||
value = syaml.load_config(value)
|
||||
existing = spack.config.get(path, scope=scope)
|
||||
|
||||
# append values to lists
|
||||
if isinstance(existing, list) and not isinstance(value, list):
|
||||
value = [value]
|
||||
|
||||
# merge value into existing
|
||||
new = spack.config.merge_yaml(existing, value)
|
||||
spack.config.set(path, new, scope)
|
||||
spack.config.add(args.path, scope=scope)
|
||||
|
||||
|
||||
def config_remove(args):
|
||||
|
|
|
|||
|
|
@ -806,6 +806,81 @@ def _config():
|
|||
config = llnl.util.lang.Singleton(_config)
|
||||
|
||||
|
||||
def add_from_file(filename, scope=None):
|
||||
"""Add updates to a config from a filename
|
||||
"""
|
||||
import spack.environment as ev
|
||||
|
||||
# Get file as config dict
|
||||
data = read_config_file(filename)
|
||||
if any(k in data for k in spack.schema.env.keys):
|
||||
data = ev.config_dict(data)
|
||||
|
||||
# update all sections from config dict
|
||||
# We have to iterate on keys to keep overrides from the file
|
||||
for section in data.keys():
|
||||
if section in section_schemas.keys():
|
||||
# Special handling for compiler scope difference
|
||||
# Has to be handled after we choose a section
|
||||
if scope is None:
|
||||
scope = default_modify_scope(section)
|
||||
|
||||
value = data[section]
|
||||
existing = get(section, scope=scope)
|
||||
new = merge_yaml(existing, value)
|
||||
|
||||
# We cannot call config.set directly (set is a type)
|
||||
config.set(section, new, scope)
|
||||
|
||||
|
||||
def add(fullpath, scope=None):
|
||||
"""Add the given configuration to the specified config scope.
|
||||
Add accepts a path. If you want to add from a filename, use add_from_file"""
|
||||
|
||||
components = process_config_path(fullpath)
|
||||
|
||||
has_existing_value = True
|
||||
path = ''
|
||||
override = False
|
||||
for idx, name in enumerate(components[:-1]):
|
||||
# First handle double colons in constructing path
|
||||
colon = '::' if override else ':' if path else ''
|
||||
path += colon + name
|
||||
if getattr(name, 'override', False):
|
||||
override = True
|
||||
else:
|
||||
override = False
|
||||
|
||||
# Test whether there is an existing value at this level
|
||||
existing = get(path, scope=scope)
|
||||
|
||||
if existing is None:
|
||||
has_existing_value = False
|
||||
# We've nested further than existing config, so we need the
|
||||
# type information for validation to know how to handle bare
|
||||
# values appended to lists.
|
||||
existing = get_valid_type(path)
|
||||
|
||||
# construct value from this point down
|
||||
value = syaml.load_config(components[-1])
|
||||
for component in reversed(components[idx + 1:-1]):
|
||||
value = {component: value}
|
||||
break
|
||||
|
||||
if has_existing_value:
|
||||
path, _, value = fullpath.rpartition(':')
|
||||
value = syaml.load_config(value)
|
||||
existing = get(path, scope=scope)
|
||||
|
||||
# append values to lists
|
||||
if isinstance(existing, list) and not isinstance(value, list):
|
||||
value = [value]
|
||||
|
||||
# merge value into existing
|
||||
new = merge_yaml(existing, value)
|
||||
config.set(path, new, scope)
|
||||
|
||||
|
||||
def get(path, default=None, scope=None):
|
||||
"""Module-level wrapper for ``Configuration.get()``."""
|
||||
return config.get(path, default, scope)
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@
|
|||
import spack.util.executable as exe
|
||||
from spack.error import SpackError
|
||||
|
||||
|
||||
#: names of profile statistics
|
||||
stat_names = pstats.Stats.sort_arg_dict_default
|
||||
|
||||
|
|
@ -358,6 +357,9 @@ def make_argument_parser(**kwargs):
|
|||
'--color', action='store', default='auto',
|
||||
choices=('always', 'never', 'auto'),
|
||||
help="when to colorize output (default: auto)")
|
||||
parser.add_argument(
|
||||
'-c', '--config', default=None, action="append", dest="config_vars",
|
||||
help="add one or more custom, one off config settings.")
|
||||
parser.add_argument(
|
||||
'-C', '--config-scope', dest='config_scopes', action='append',
|
||||
metavar='DIR', help="add a custom configuration scope")
|
||||
|
|
@ -463,6 +465,10 @@ def setup_main_options(args):
|
|||
tty.warn("You asked for --insecure. Will NOT check SSL certificates.")
|
||||
spack.config.set('config:verify_ssl', False, scope='command_line')
|
||||
|
||||
# Use the spack config command to handle parsing the config strings
|
||||
for config_var in (args.config_vars or []):
|
||||
spack.config.add(path=config_var, scope="command_line")
|
||||
|
||||
# when to use color (takes always, auto, or never)
|
||||
color.set_color_when(args.color)
|
||||
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ def test_get_config_scope_merged(mock_low_high_config):
|
|||
|
||||
def test_config_edit():
|
||||
"""Ensure `spack config edit` edits the right paths."""
|
||||
|
||||
dms = spack.config.default_modify_scope('compilers')
|
||||
dms_path = spack.config.config.scopes[dms].path
|
||||
user_path = spack.config.config.scopes['user'].path
|
||||
|
|
@ -204,20 +205,27 @@ def test_config_add_override_leaf(mutable_empty_config):
|
|||
|
||||
|
||||
def test_config_add_update_dict(mutable_empty_config):
|
||||
config('add', 'packages:all:compiler:[gcc]')
|
||||
config('add', 'packages:all:version:1.0.0')
|
||||
config('add', 'packages:all:version:[1.0.0]')
|
||||
output = config('get', 'packages')
|
||||
|
||||
expected = """packages:
|
||||
all:
|
||||
compiler: [gcc]
|
||||
version:
|
||||
- 1.0.0
|
||||
"""
|
||||
|
||||
expected = 'packages:\n all:\n version: [1.0.0]\n'
|
||||
assert output == expected
|
||||
|
||||
|
||||
def test_config_with_c_argument(mutable_empty_config):
|
||||
|
||||
# I don't know how to add a spack argument to a Spack Command, so we test this way
|
||||
config_file = 'config:install_root:root:/path/to/config.yaml'
|
||||
parser = spack.main.make_argument_parser()
|
||||
args = parser.parse_args(['-c', config_file])
|
||||
assert config_file in args.config_vars
|
||||
|
||||
# Add the path to the config
|
||||
config("add", args.config_vars[0], scope='command_line')
|
||||
output = config("get", 'config')
|
||||
assert "config:\n install_root:\n - root: /path/to/config.yaml" in output
|
||||
|
||||
|
||||
def test_config_add_ordered_dict(mutable_empty_config):
|
||||
config('add', 'mirrors:first:/path/to/first')
|
||||
config('add', 'mirrors:second:/path/to/second')
|
||||
|
|
|
|||
|
|
@ -258,6 +258,37 @@ def test_write_to_same_priority_file(mock_low_high_config, compiler_specs):
|
|||
repos_low = {'repos': ["/some/path"]}
|
||||
repos_high = {'repos': ["/some/other/path"]}
|
||||
|
||||
# Test setting config values via path in filename
|
||||
|
||||
|
||||
def test_add_config_path():
|
||||
|
||||
# Try setting a new install tree root
|
||||
path = "config:install_tree:root:/path/to/config.yaml"
|
||||
spack.config.add(path, scope="command_line")
|
||||
set_value = spack.config.get('config')['install_tree']['root']
|
||||
assert set_value == '/path/to/config.yaml'
|
||||
|
||||
# Now a package:all setting
|
||||
path = "packages:all:compiler:[gcc]"
|
||||
spack.config.add(path, scope="command_line")
|
||||
compilers = spack.config.get('packages')['all']['compiler']
|
||||
assert "gcc" in compilers
|
||||
|
||||
|
||||
def test_add_config_filename(mock_low_high_config, tmpdir):
|
||||
|
||||
config_yaml = tmpdir.join('config-filename.yaml')
|
||||
config_yaml.ensure()
|
||||
with config_yaml.open('w') as f:
|
||||
syaml.dump_config(config_low, f)
|
||||
|
||||
spack.config.add_from_file(str(config_yaml), scope="low")
|
||||
assert "build_stage" in spack.config.get('config')
|
||||
build_stages = spack.config.get('config')['build_stage']
|
||||
for stage in config_low['config']['build_stage']:
|
||||
assert stage in build_stages
|
||||
|
||||
|
||||
# repos
|
||||
def test_write_list_in_memory(mock_low_high_config):
|
||||
|
|
|
|||
|
|
@ -331,7 +331,7 @@ _spacktivate() {
|
|||
_spack() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help -H --all-help --color -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
|
||||
SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
|
||||
else
|
||||
SPACK_COMPREPLY="activate add arch blame build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop docs edit env extensions external fetch find flake8 gc gpg graph help info install license list load location log-parse maintainers mark mirror module patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
|
||||
fi
|
||||
|
|
|
|||
Loading…
Reference in a new issue