From c9ea9575637ed71ba5dc055609544afcb7429852 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 13 Apr 2018 23:10:25 -0700 Subject: [PATCH] config: create internal config scope for commands to use. --- lib/spack/spack/compilers/__init__.py | 7 +-- lib/spack/spack/config.py | 56 ++++++++++++++++++--- lib/spack/spack/fetch_strategy.py | 1 + lib/spack/spack/hooks/yaml_version_check.py | 2 +- lib/spack/spack/test/config.py | 34 ++++++++++++- 5 files changed, 87 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index ed6d4c8db2..eb637259b5 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -117,10 +117,11 @@ def init_compiler_config(): def compiler_config_files(): config_files = list() config = spack.config.get_configuration() - for scope in config.scopes: - compiler_config = config.get_config('compilers', scope=scope) + for scope in config.file_scopes: + name = scope.name + compiler_config = config.get_config('compilers', scope=name) if compiler_config: - config_files.append(config.get_config_filename(scope, 'compilers')) + config_files.append(config.get_config_filename(name, 'compilers')) return config_files diff --git a/lib/spack/spack/config.py b/lib/spack/spack/config.py index 1a32a2ddd4..d622f0416d 100644 --- a/lib/spack/spack/config.py +++ b/lib/spack/spack/config.py @@ -109,6 +109,10 @@ scopes_metavar = '{defaults,system,site,user}[/PLATFORM]' +#: config scopes only used by Spack internally +internal_scopes = ['commands'] + + def _extend_with_default(validator_class): """Add support for the 'default' attr for properties and patternProperties. @@ -199,6 +203,37 @@ def __repr__(self): return '' % (self.name, self.path) +class InternalConfigScope(ConfigScope): + """An internal configuration scope that is not persisted to a file. + + This is for spack internal use so that command-line options and + config file settings are accessed the same way, and Spack can easily + override settings from files. + """ + def __init__(self, name): + self.name = name + self.sections = syaml.syaml_dict() + + def get_section_filename(self, section): + raise NotImplementedError( + "Cannot get filename for InternalConfigScope.") + + def get_section(self, section): + """Just reads from an internal dictionary.""" + if section not in self.sections: + self.sections[section] = None + return self.sections[section] + + def write_section(self, section): + """This only validates, as the data is already in memory.""" + data = self.get_section(section) + if data is not None: + _validate_section(data, section_schemas[section]) + + def __repr__(self): + return '' % self.name + + class Configuration(object): """A full Spack configuration, from a hierarchy of config files. @@ -226,10 +261,15 @@ def pop_scope(self): name, scope = self.scopes.popitem(last=True) return scope + @property + def file_scopes(self): + """List of scopes with an associated file (non-internal scopes).""" + return [s for s in self.scopes.values() + if not isinstance(s, InternalConfigScope)] + def highest_precedence_scope(self): - """Get the scope with highest precedence (prefs will override others). - """ - return list(self.scopes.values())[-1] + """Non-internal scope with highest precedence.""" + return next(reversed(self.file_scopes), None) def _validate_scope(self, scope): """Ensure that scope is valid in this configuration. @@ -371,17 +411,19 @@ def get_configuration(): # configuration directory. platform = spack.architecture.platform().name - scopes = [] + _configuration = Configuration() for name, path in configuration_paths: # add the regular scope - scopes.append(ConfigScope(name, path)) + _configuration.push_scope(ConfigScope(name, path)) # add platform-specific scope plat_name = '%s/%s' % (name, platform) plat_path = os.path.join(path, platform) - scopes.append(ConfigScope(plat_name, plat_path)) + _configuration.push_scope(ConfigScope(plat_name, plat_path)) - _configuration = Configuration(*scopes) + # we make a special scope for spack commands so that they can + # override configuration options. + _configuration.push_scope(InternalConfigScope('commands')) return _configuration diff --git a/lib/spack/spack/fetch_strategy.py b/lib/spack/spack/fetch_strategy.py index 04e42a9f0c..779a4315fe 100644 --- a/lib/spack/spack/fetch_strategy.py +++ b/lib/spack/spack/fetch_strategy.py @@ -52,6 +52,7 @@ from llnl.util.filesystem import working_dir, mkdirp, join_path import spack +import spack.config import spack.error import spack.util.crypto as crypto import spack.util.pattern as pattern diff --git a/lib/spack/spack/hooks/yaml_version_check.py b/lib/spack/spack/hooks/yaml_version_check.py index ec695319d1..a2bccf1fb1 100644 --- a/lib/spack/spack/hooks/yaml_version_check.py +++ b/lib/spack/spack/hooks/yaml_version_check.py @@ -38,7 +38,7 @@ def pre_run(): def check_compiler_yaml_version(): config = spack.config.get_configuration() - for scope in config: + for scope in config.file_scopes: file_name = os.path.join(scope.path, 'compilers.yaml') data = None if os.path.isfile(file_name): diff --git a/lib/spack/spack/test/config.py b/lib/spack/spack/test/config.py index 17e749d6a2..41b02d493e 100644 --- a/lib/spack/spack/test/config.py +++ b/lib/spack/spack/test/config.py @@ -64,9 +64,10 @@ def config(tmpdir): real_configuration = spack.config._configuration scopes = [spack.config.ConfigScope(name, str(tmpdir.join(name))) for name in ['low', 'high']] - spack.config._configuration = spack.config.Configuration(*scopes) + config = spack.config.Configuration(*scopes) + spack.config._configuration = config - yield + yield config spack.config._configuration = real_configuration @@ -420,6 +421,35 @@ def test_read_config_override_list(config, write_config_file): } +def test_internal_config_update(config, write_config_file): + write_config_file('config', config_low, 'low') + + before = config.get_config('config') + assert before['install_tree'] == 'install_tree_path' + + # add an internal configuration scope + scope = spack.config.InternalConfigScope('commands') + assert 'InternalConfigScope' in repr(scope) + + config.push_scope(scope) + + command_config = config.get_config('config', scope='commands') + command_config['install_tree'] = 'foo/bar' + + config.update_config('config', command_config, scope='commands') + + after = config.get_config('config') + assert after['install_tree'] == 'foo/bar' + + +def test_internal_config_filename(config, write_config_file): + write_config_file('config', config_low, 'low') + config.push_scope(spack.config.InternalConfigScope('commands')) + + with pytest.raises(NotImplementedError): + config.get_config_filename('commands', 'config') + + def test_keys_are_ordered(): expected_order = (