concretizer: optimize microarchitectures, constrained by compiler support

Weight microarchitectures and prefers more rercent ones.  Also disallow
nodes where the compiler does not support the selected target.

We should revisit this at some point as it seems like if I play around
with the compiler support for different architectures, the solver runs
very slowly.  See notes in comments -- the bad case was gcc supporting
broadwell and skylake with clang maxing out at haswell.

lib/spack/spack/solver/asp.py

@@ -265,6 +265,8 @@ def spec_versions(self, spec):
 
     def available_compilers(self):
         """Facts about available compilers."""
+
+        self.h2("Available compilers")
         compilers = spack.compilers.all_compiler_specs()
 
         compiler_versions = collections.defaultdict(lambda: set())
@@ -438,18 +440,18 @@ def spec_clauses(self, spec, body=False):
         # TODO: do this with consistent suffixes.
         class Head(object):
             node = fn.node
-            arch_platform = fn.arch_platform_set
+            node_platform = fn.node_platform_set
-            arch_os = fn.arch_os_set
+            node_os = fn.node_os_set
-            arch_target = fn.arch_target_set
+            node_target = fn.node_target_set
             variant = fn.variant_set
             node_compiler = fn.node_compiler
             node_compiler_version = fn.node_compiler_version
 
         class Body(object):
             node = fn.node
-            arch_platform = fn.arch_platform
+            node_platform = fn.node_platform
-            arch_os = fn.arch_os
+            node_os = fn.node_os
-            arch_target = fn.arch_target
+            node_target = fn.node_target
             variant = fn.variant_value
             node_compiler = fn.node_compiler
             node_compiler_version = fn.node_compiler_version
@@ -466,11 +468,11 @@ class Body(object):
         arch = spec.architecture
         if arch:
             if arch.platform:
-                clauses.append(f.arch_platform(spec.name, arch.platform))
+                clauses.append(f.node_platform(spec.name, arch.platform))
             if arch.os:
-                clauses.append(f.arch_os(spec.name, arch.os))
+                clauses.append(f.node_os(spec.name, arch.os))
             if arch.target:
-                clauses.append(f.arch_target(spec.name, arch.target))
+                clauses.append(f.node_target(spec.name, arch.target))
 
         # variants
         for vname, variant in sorted(spec.variants.items()):
@@ -528,13 +530,83 @@ def build_version_dict(self, possible_pkgs, specs):
                 if dep.versions.concrete:
                     self.possible_versions[dep.name].add(dep.version)
 
+    def _supported_targets(self, compiler, targets):
+        """Get a list of which targets are supported by the compiler.
+
+        Results are ordered most to least recent.
+        """
+        supported = []
+
+        for target in targets:
+            compiler_info = target.compilers.get(compiler.name)
+            if not compiler_info:
+                # if we don't know, we assume it's supported and leave it
+                # to the user to debug
+                supported.append(target)
+                continue
+
+            if not isinstance(compiler_info, list):
+                compiler_info = [compiler_info]
+
+            for info in compiler_info:
+                version = ver(info['versions'])
+                if compiler.version.satisfies(version):
+                    supported.append(target)
+
+        return sorted(supported, reverse=True)
+
     def arch_defaults(self):
         """Add facts about the default architecture for a package."""
         self.h2('Default architecture')
-        default_arch = spack.spec.ArchSpec(spack.architecture.sys_type())
+        default = spack.spec.ArchSpec(spack.architecture.sys_type())
-        self.fact(fn.arch_platform_default(default_arch.platform))
+        self.fact(fn.node_platform_default(default.platform))
-        self.fact(fn.arch_os_default(default_arch.os))
+        self.fact(fn.node_os_default(default.os))
-        self.fact(fn.arch_target_default(default_arch.target))
+        self.fact(fn.node_target_default(default.target))
+
+        uarch = default.target.microarchitecture
+
+        self.h2('Target compatibility')
+
+        # listing too many targets can be slow, at least with our current
+        # encoding. To reduce the number of options to consider, only
+        # consider the *best* target that each compiler supports, along
+        # with the family.
+        compatible_targets = [uarch] + uarch.ancestors
+        compilers = self.compilers_for_default_arch()
+
+        # this loop can be used to limit the number of targets
+        # considered. Right now we consider them all, but it seems that
+        # many targets can make things slow.
+        # TODO: investigate this.
+        best_targets = set([uarch.family.name])
+        for compiler in compilers:
+            supported = self._supported_targets(compiler, compatible_targets)
+            if not supported:
+                continue
+
+            for target in supported:
+                best_targets.add(target.name)
+                self.fact(fn.compiler_supports_target(
+                    compiler.name, compiler.version, target.name))
+                self.fact(fn.compiler_supports_target(
+                    compiler.name, compiler.version, uarch.family.name))
+
+        i = 0
+        for target in compatible_targets:
+            self.fact(fn.target(target.name))
+            self.fact(fn.target_family(target.name, target.family.name))
+            for parent in sorted(target.parents):
+                self.fact(fn.target_parent(target.name, parent.name))
+
+            # prefer best possible targets; weight others poorly so
+            # they're not used unless set explicitly
+            if target.name in best_targets:
+                self.fact(fn.target_weight(target.name, i))
+                i += 1
+            else:
+                self.fact(fn.target_weight(target.name, 100))
+
+            self.out.write("\n")
 
     def virtual_providers(self):
         self.h2("Virtual providers")
@@ -556,20 +628,13 @@ def generate_asp_program(self, specs):
             specs (list): list of Specs to solve
         """
         # get list of all possible dependencies
-        pkg_names = set(spec.fullname for spec in specs)
-
-        possible = set()
         self.possible_virtuals = set()
-        for name in pkg_names:
+        possible = spack.package.possible_dependencies(
-            pkg = spack.repo.path.get_pkg_class(name)
+            *specs,
-            possible.update(
-                pkg.possible_dependencies(
             virtuals=self.possible_virtuals,
             deptype=("build", "link", "run")
         )
-            )
+        pkgs = set(possible)
-
-        pkgs = set(possible) | set(pkg_names)
 
         # get possible compilers
         self.possible_compilers = self.compilers_for_default_arch()
@@ -623,13 +688,13 @@ def _arch(self, pkg):
             self._specs[pkg].architecture = arch
         return arch
 
-    def arch_platform(self, pkg, platform):
+    def node_platform(self, pkg, platform):
         self._arch(pkg).platform = platform
 
-    def arch_os(self, pkg, os):
+    def node_os(self, pkg, os):
         self._arch(pkg).os = os
 
-    def arch_target(self, pkg, target):
+    def node_target(self, pkg, target):
         self._arch(pkg).target = target
 
     def variant_value(self, pkg, name, value):

lib/spack/spack/solver/concretize.lp

@@ -107,40 +107,47 @@ variant_not_default(P, V, X, 0)
 #defined variant_single_value/2.
 
 %-----------------------------------------------------------------------------
-% Architecture semantics
+% Platform/OS semantics
 %-----------------------------------------------------------------------------
-
+% one platform, os per node
-% one platform, os, target per node.
+% TODO: convert these to use optimization, like targets.
-1 { arch_platform(P, A) : arch_platform(P, A) } 1 :- node(P).
+1 { node_platform(P, A) : node_platform(P, A) } 1 :- node(P).
-1 { arch_os(P, A) : arch_os(P, A) } 1             :- node(P).
+1 { node_os(P, A) : node_os(P, A) } 1 :- node(P).
-1 { arch_target(P, T) : arch_target(P, T) } 1     :- node(P).
 
 % arch fields for pkg P are set if set to anything
-arch_platform_set(P) :- arch_platform_set(P, _).
+node_platform_set(P) :- node_platform_set(P, _).
-arch_os_set(P) :- arch_os_set(P, _).
+node_os_set(P) :- node_os_set(P, _).
-arch_target_set(P) :- arch_target_set(P, _).
+
+% if no platform/os is set, fall back to the defaults
+node_platform(P, A)
+    :- node(P), not node_platform_set(P), node_platform_default(A).
+node_os(P, A) :- node(P), not node_os_set(P), node_os_default(A).
+
+% setting os/platform on a node is a hard constraint
+node_platform(P, A) :- node(P), node_platform_set(P, A).
+node_os(P, A) :- node(P), node_os_set(P, A).
 
 % avoid info warnings (see variants)
-#defined arch_platform_set/2.
+#defined node_platform_set/2.
-#defined arch_os_set/2.
+#defined node_os_set/2.
-#defined arch_target_set/2.
 
-% if architecture value is set, it's the value
+%-----------------------------------------------------------------------------
-arch_platform(P, A) :- node(P), arch_platform_set(P, A).
+% Target semantics
-arch_os(P, A) :- node(P), arch_os_set(P, A).
+%-----------------------------------------------------------------------------
-arch_target(P, A) :- node(P), arch_target_set(P, A).
+% one target per node -- optimization will pick the "best" one
+1 { node_target(P, T) : target(T) } 1 :- node(P).
 
-% if no architecture is set, fall back to the default architecture value.
+% can't use targets on node if the compiler for the node doesn't support them
-arch_platform(P, A) :- node(P), not arch_platform_set(P),
+:- node_target(P, T), not compiler_supports_target(C, V, T),
-                       arch_platform_default(A).
+   node_compiler(P, C), node_compiler_version(P, C, V).
-arch_os(P, A)       :- node(P), not arch_os_set(P), arch_os_default(A).
-arch_target(P, A)   :- node(P), not arch_target_set(P), arch_target_default(A).
 
-% propagate platform, os, target downwards
+% if a target is set explicitly, respect it
-% TODO: handle multiple dependents and arch compatibility
+node_target(P, T) :- node(P), node_target_set(P, T).
-arch_platform_set(D, A) :- node(D), depends_on(P, D), arch_platform_set(P, A).
+
-arch_os_set(D, A) :- node(D), depends_on(P, D), arch_os_set(P, A).
+% each node has the weight of its assigned target
-arch_target_set(D, A) :- node(D), depends_on(P, D), arch_target_set(P, A).
+node_target_weight(P, N) :- node(P), node_target(P, T), target_weight(T, N).
+
+#defined node_target_set/2.
 
 %-----------------------------------------------------------------------------
 % Compiler semantics
@@ -212,14 +219,21 @@ root(D, 2) :- root(D), node(D).
 root(D, 1) :- not root(D), node(D).
 
 % prefer default variants
-#minimize { N*R@5,P,V,X : variant_not_default(P, V, X, N), root(P, R) }.
+#minimize { N*R@10,P,V,X : variant_not_default(P, V, X, N), root(P, R) }.
 
 % pick most preferred virtual providers
-#minimize{ N*R@4,D : provider_weight(D, N), root(P, R) }.
+#minimize{ N*R@9,D : provider_weight(D, N), root(P, R) }.
 
 % prefer more recent versions.
-#minimize{ N@3,P,V : version_weight(P, V, N) }.
+#minimize{ N@8,P,V : version_weight(P, V, N) }.
 
 % compiler preferences
-#maximize{ N@2,P : compiler_match(P, N) }.
+#maximize{ N@7,P : compiler_match(P, N) }.
-#minimize{ N@1,P : compiler_weight(P, N) }.
+#minimize{ N@6,P : compiler_weight(P, N) }.
+
+% fastest target for node
+
+% TODO: if these are slightly different by compiler (e.g., skylake is
+% best, gcc supports skylake and broadweell, clang's best is haswell)
+% things seem to get really slow.
+#minimize{ N@5,P : node_target_weight(P, N) }.

lib/spack/spack/solver/display.lp

@@ -7,8 +7,8 @@
 #show depends_on/3.
 #show version/2.
 #show variant_value/3.
-#show arch_platform/2.
+#show node_platform/2.
-#show arch_os/2.
+#show node_os/2.
-#show arch_target/2.
+#show node_target/2.
 #show node_compiler/2.
 #show node_compiler_version/3.