Config prefer upstream (#21487)

This allows for quickly configuring a spack install/env to use upstream packages by default. This is particularly important when upstreaming from a set of officially supported spack installs on a production cluster. By configuring such that package preferences match the upstream, you ensure maximal reuse of existing package installations.

lib/spack/spack/cmd/config.py

@@ -17,6 +17,8 @@
 import spack.schema.packages
 import spack.util.spack_yaml as syaml
 from spack.util.editor import editor
+import spack.store
+import spack.repo
 
 description = "get and set configuration options"
 section = "config"
@@ -73,6 +75,16 @@ def setup_parser(subparser):
         help="file from which to set all config values"
     )
 
+    prefer_upstream_parser = sp.add_parser(
+        'prefer-upstream',
+        help='set package preferences from upstream')
+
+    prefer_upstream_parser.add_argument(
+        '--local', action='store_true', default=False,
+        help="Set packages preferences based on local installs, rather "
+             "than upstream."
+    )
+
     remove_parser = sp.add_parser('remove', aliases=['rm'],
                                   help='remove configuration parameters')
     remove_parser.add_argument(
@@ -431,6 +443,79 @@ def config_revert(args):
         tty.msg(msg.format(cfg_file))
 
 
+def config_prefer_upstream(args):
+    """Generate a packages config based on the configuration of all upstream
+    installs."""
+
+    scope = args.scope
+    if scope is None:
+        scope = spack.config.default_modify_scope('packages')
+
+    all_specs = set(spack.store.db.query(installed=True))
+    local_specs = set(spack.store.db.query_local(installed=True))
+    pref_specs = local_specs if args.local else all_specs - local_specs
+
+    conflicting_variants = set()
+
+    pkgs = {}
+    for spec in pref_specs:
+        # Collect all the upstream compilers and versions for this package.
+        pkg = pkgs.get(spec.name, {
+            'version': [],
+            'compiler': [],
+        })
+        pkgs[spec.name] = pkg
+
+        # We have no existing variant if this is our first added version.
+        existing_variants = pkg.get('variants',
+                                    None if not pkg['version'] else '')
+
+        version = spec.version.string
+        if version not in pkg['version']:
+            pkg['version'].append(version)
+
+        compiler = str(spec.compiler)
+        if compiler not in pkg['compiler']:
+            pkg['compiler'].append(compiler)
+
+        # Get and list all the variants that differ from the default.
+        variants = []
+        for var_name, variant in spec.variants.items():
+            if (var_name in ['patches']
+                    or var_name not in spec.package.variants):
+                continue
+
+            if variant.value != spec.package.variants[var_name].default:
+                variants.append(str(variant))
+        variants.sort()
+        variants = ' '.join(variants)
+
+        if spec.name not in conflicting_variants:
+            # Only specify the variants if there's a single variant
+            # set across all versions/compilers.
+            if existing_variants is not None and existing_variants != variants:
+                conflicting_variants.add(spec.name)
+                pkg.pop('variants', None)
+            elif variants:
+                pkg['variants'] = variants
+
+    if conflicting_variants:
+        tty.warn(
+            "The following packages have multiple conflicting upstream "
+            "specs. You may have to specify, by "
+            "concretized hash, which spec you want when building "
+            "packages that depend on them:\n - {0}"
+            .format("\n - ".join(sorted(conflicting_variants))))
+
+    # Simply write the config to the specified file.
+    existing = spack.config.get('packages', scope=scope)
+    new = spack.config.merge_yaml(existing, pkgs)
+    spack.config.set('packages', new, scope)
+    config_file = spack.config.config.get_config_filename(scope, section)
+
+    tty.msg("Updated config at {0}".format(config_file))
+
+
 def config(parser, args):
     action = {
         'get': config_get,
@@ -441,6 +526,7 @@ def config(parser, args):
         'rm': config_remove,
         'remove': config_remove,
         'update': config_update,
-        'revert': config_revert
+        'revert': config_revert,
+        'prefer-upstream': config_prefer_upstream,
     }
     action[args.config_command](args)

lib/spack/spack/config.py

@@ -553,7 +553,7 @@ def get_config(self, section, scope=None):
 
         If ``scope`` is ``None`` or not provided, return the merged contents
         of all of Spack's configuration scopes.  If ``scope`` is provided,
-        return only the confiugration as specified in that scope.
+        return only the configuration as specified in that scope.
 
         This off the top-level name from the YAML section.  That is, for a
         YAML config file that looks like this::

lib/spack/spack/test/cmd/config.py

@@ -11,6 +11,9 @@
 import spack.environment as ev
 import spack.main
 import spack.util.spack_yaml as syaml
+import spack.spec
+import spack.database
+import spack.store
 
 config = spack.main.SpackCommand('config')
 env = spack.main.SpackCommand('env')
@@ -645,3 +648,50 @@ def check_config_updated(data):
     assert isinstance(data['install_tree'], dict)
     assert data['install_tree']['root'] == '/fake/path'
     assert data['install_tree']['projections'] == {'all': '{name}-{version}'}
+
+
+def test_config_prefer_upstream(tmpdir_factory, install_mockery, mock_fetch,
+                                mutable_config, gen_mock_layout, monkeypatch):
+    """Check that when a dependency package is recorded as installed in
+       an upstream database that it is not reinstalled.
+    """
+
+    mock_db_root = str(tmpdir_factory.mktemp('mock_db_root'))
+    prepared_db = spack.database.Database(mock_db_root)
+
+    upstream_layout = gen_mock_layout('/a/')
+
+    for spec in [
+            'hdf5 +mpi',
+            'hdf5 ~mpi',
+            'boost+debug~icu+graph',
+            'dependency-install',
+            'patch']:
+        dep = spack.spec.Spec(spec)
+        dep.concretize()
+        prepared_db.add(dep, upstream_layout)
+
+    downstream_db_root = str(
+        tmpdir_factory.mktemp('mock_downstream_db_root'))
+    db_for_test = spack.database.Database(
+        downstream_db_root, upstream_dbs=[prepared_db])
+    monkeypatch.setattr(spack.store, 'db', db_for_test)
+
+    output = config('prefer-upstream')
+    scope = spack.config.default_modify_scope('packages')
+    cfg_file = spack.config.config.get_config_filename(scope, 'packages')
+    packages = syaml.load(open(cfg_file))['packages']
+
+    # Make sure only the non-default variants are set.
+    assert packages['boost'] == {
+        'compiler': ['gcc@4.5.0'],
+        'variants': '+debug +graph',
+        'version': ['1.63.0']}
+    assert packages['dependency-install'] == {
+        'compiler': ['gcc@4.5.0'], 'version': ['2.0']}
+    # Ensure that neither variant gets listed for hdf5, since they conflict
+    assert packages['hdf5'] == {
+        'compiler': ['gcc@4.5.0'], 'version': ['2.3']}
+
+    # Make sure a message about the conflicting hdf5's was given.
+    assert '- hdf5' in output

share/spack/spack-completion.bash

@@ -587,7 +587,7 @@ _spack_config() {
     then
         SPACK_COMPREPLY="-h --help --scope"
     else
-        SPACK_COMPREPLY="get blame edit list add remove rm update revert"
+        SPACK_COMPREPLY="get blame edit list add prefer-upstream remove rm update revert"
     fi
 }
 
@@ -631,6 +631,10 @@ _spack_config_add() {
     fi
 }
 
+_spack_config_prefer_upstream() {
+    SPACK_COMPREPLY="-h --help --local"
+}
+
 _spack_config_remove() {
     if $list_options
     then