concretizer: optimize microarchitectures, constrained by compiler support
Weight microarchitectures and prefers more rercent ones. Also disallow nodes where the compiler does not support the selected target. We should revisit this at some point as it seems like if I play around with the compiler support for different architectures, the solver runs very slowly. See notes in comments -- the bad case was gcc supporting broadwell and skylake with clang maxing out at haswell.
This commit is contained in:
Todd Gamblin
parent
71726a9b33
commit
5185ed1d28
3 changed files with 142 additions and 63 deletions
|
|
@ -265,6 +265,8 @@ def spec_versions(self, spec):
|
|||
|
||||
def available_compilers(self):
|
||||
"""Facts about available compilers."""
|
||||
|
||||
self.h2("Available compilers")
|
||||
compilers = spack.compilers.all_compiler_specs()
|
||||
|
||||
compiler_versions = collections.defaultdict(lambda: set())
|
||||
|
|
@ -438,18 +440,18 @@ def spec_clauses(self, spec, body=False):
|
|||
# TODO: do this with consistent suffixes.
|
||||
class Head(object):
|
||||
node = fn.node
|
||||
arch_platform = fn.arch_platform_set
|
||||
arch_os = fn.arch_os_set
|
||||
arch_target = fn.arch_target_set
|
||||
node_platform = fn.node_platform_set
|
||||
node_os = fn.node_os_set
|
||||
node_target = fn.node_target_set
|
||||
variant = fn.variant_set
|
||||
node_compiler = fn.node_compiler
|
||||
node_compiler_version = fn.node_compiler_version
|
||||
|
||||
class Body(object):
|
||||
node = fn.node
|
||||
arch_platform = fn.arch_platform
|
||||
arch_os = fn.arch_os
|
||||
arch_target = fn.arch_target
|
||||
node_platform = fn.node_platform
|
||||
node_os = fn.node_os
|
||||
node_target = fn.node_target
|
||||
variant = fn.variant_value
|
||||
node_compiler = fn.node_compiler
|
||||
node_compiler_version = fn.node_compiler_version
|
||||
|
|
@ -466,11 +468,11 @@ class Body(object):
|
|||
arch = spec.architecture
|
||||
if arch:
|
||||
if arch.platform:
|
||||
clauses.append(f.arch_platform(spec.name, arch.platform))
|
||||
clauses.append(f.node_platform(spec.name, arch.platform))
|
||||
if arch.os:
|
||||
clauses.append(f.arch_os(spec.name, arch.os))
|
||||
clauses.append(f.node_os(spec.name, arch.os))
|
||||
if arch.target:
|
||||
clauses.append(f.arch_target(spec.name, arch.target))
|
||||
clauses.append(f.node_target(spec.name, arch.target))
|
||||
|
||||
# variants
|
||||
for vname, variant in sorted(spec.variants.items()):
|
||||
|
|
@ -528,13 +530,83 @@ def build_version_dict(self, possible_pkgs, specs):
|
|||
if dep.versions.concrete:
|
||||
self.possible_versions[dep.name].add(dep.version)
|
||||
|
||||
def _supported_targets(self, compiler, targets):
|
||||
"""Get a list of which targets are supported by the compiler.
|
||||
|
||||
Results are ordered most to least recent.
|
||||
"""
|
||||
supported = []
|
||||
|
||||
for target in targets:
|
||||
compiler_info = target.compilers.get(compiler.name)
|
||||
if not compiler_info:
|
||||
# if we don't know, we assume it's supported and leave it
|
||||
# to the user to debug
|
||||
supported.append(target)
|
||||
continue
|
||||
|
||||
if not isinstance(compiler_info, list):
|
||||
compiler_info = [compiler_info]
|
||||
|
||||
for info in compiler_info:
|
||||
version = ver(info['versions'])
|
||||
if compiler.version.satisfies(version):
|
||||
supported.append(target)
|
||||
|
||||
return sorted(supported, reverse=True)
|
||||
|
||||
def arch_defaults(self):
|
||||
"""Add facts about the default architecture for a package."""
|
||||
self.h2('Default architecture')
|
||||
default_arch = spack.spec.ArchSpec(spack.architecture.sys_type())
|
||||
self.fact(fn.arch_platform_default(default_arch.platform))
|
||||
self.fact(fn.arch_os_default(default_arch.os))
|
||||
self.fact(fn.arch_target_default(default_arch.target))
|
||||
default = spack.spec.ArchSpec(spack.architecture.sys_type())
|
||||
self.fact(fn.node_platform_default(default.platform))
|
||||
self.fact(fn.node_os_default(default.os))
|
||||
self.fact(fn.node_target_default(default.target))
|
||||
|
||||
uarch = default.target.microarchitecture
|
||||
|
||||
self.h2('Target compatibility')
|
||||
|
||||
# listing too many targets can be slow, at least with our current
|
||||
# encoding. To reduce the number of options to consider, only
|
||||
# consider the *best* target that each compiler supports, along
|
||||
# with the family.
|
||||
compatible_targets = [uarch] + uarch.ancestors
|
||||
compilers = self.compilers_for_default_arch()
|
||||
|
||||
# this loop can be used to limit the number of targets
|
||||
# considered. Right now we consider them all, but it seems that
|
||||
# many targets can make things slow.
|
||||
# TODO: investigate this.
|
||||
best_targets = set([uarch.family.name])
|
||||
for compiler in compilers:
|
||||
supported = self._supported_targets(compiler, compatible_targets)
|
||||
if not supported:
|
||||
continue
|
||||
|
||||
for target in supported:
|
||||
best_targets.add(target.name)
|
||||
self.fact(fn.compiler_supports_target(
|
||||
compiler.name, compiler.version, target.name))
|
||||
self.fact(fn.compiler_supports_target(
|
||||
compiler.name, compiler.version, uarch.family.name))
|
||||
|
||||
i = 0
|
||||
for target in compatible_targets:
|
||||
self.fact(fn.target(target.name))
|
||||
self.fact(fn.target_family(target.name, target.family.name))
|
||||
for parent in sorted(target.parents):
|
||||
self.fact(fn.target_parent(target.name, parent.name))
|
||||
|
||||
# prefer best possible targets; weight others poorly so
|
||||
# they're not used unless set explicitly
|
||||
if target.name in best_targets:
|
||||
self.fact(fn.target_weight(target.name, i))
|
||||
i += 1
|
||||
else:
|
||||
self.fact(fn.target_weight(target.name, 100))
|
||||
|
||||
self.out.write("\n")
|
||||
|
||||
def virtual_providers(self):
|
||||
self.h2("Virtual providers")
|
||||
|
|
@ -556,20 +628,13 @@ def generate_asp_program(self, specs):
|
|||
specs (list): list of Specs to solve
|
||||
"""
|
||||
# get list of all possible dependencies
|
||||
pkg_names = set(spec.fullname for spec in specs)
|
||||
|
||||
possible = set()
|
||||
self.possible_virtuals = set()
|
||||
for name in pkg_names:
|
||||
pkg = spack.repo.path.get_pkg_class(name)
|
||||
possible.update(
|
||||
pkg.possible_dependencies(
|
||||
virtuals=self.possible_virtuals,
|
||||
deptype=("build", "link", "run")
|
||||
)
|
||||
)
|
||||
|
||||
pkgs = set(possible) | set(pkg_names)
|
||||
possible = spack.package.possible_dependencies(
|
||||
*specs,
|
||||
virtuals=self.possible_virtuals,
|
||||
deptype=("build", "link", "run")
|
||||
)
|
||||
pkgs = set(possible)
|
||||
|
||||
# get possible compilers
|
||||
self.possible_compilers = self.compilers_for_default_arch()
|
||||
|
|
@ -623,13 +688,13 @@ def _arch(self, pkg):
|
|||
self._specs[pkg].architecture = arch
|
||||
return arch
|
||||
|
||||
def arch_platform(self, pkg, platform):
|
||||
def node_platform(self, pkg, platform):
|
||||
self._arch(pkg).platform = platform
|
||||
|
||||
def arch_os(self, pkg, os):
|
||||
def node_os(self, pkg, os):
|
||||
self._arch(pkg).os = os
|
||||
|
||||
def arch_target(self, pkg, target):
|
||||
def node_target(self, pkg, target):
|
||||
self._arch(pkg).target = target
|
||||
|
||||
def variant_value(self, pkg, name, value):
|
||||
|
|
|
|||
|
|
@ -107,40 +107,47 @@ variant_not_default(P, V, X, 0)
|
|||
#defined variant_single_value/2.
|
||||
|
||||
%-----------------------------------------------------------------------------
|
||||
% Architecture semantics
|
||||
% Platform/OS semantics
|
||||
%-----------------------------------------------------------------------------
|
||||
|
||||
% one platform, os, target per node.
|
||||
1 { arch_platform(P, A) : arch_platform(P, A) } 1 :- node(P).
|
||||
1 { arch_os(P, A) : arch_os(P, A) } 1 :- node(P).
|
||||
1 { arch_target(P, T) : arch_target(P, T) } 1 :- node(P).
|
||||
% one platform, os per node
|
||||
% TODO: convert these to use optimization, like targets.
|
||||
1 { node_platform(P, A) : node_platform(P, A) } 1 :- node(P).
|
||||
1 { node_os(P, A) : node_os(P, A) } 1 :- node(P).
|
||||
|
||||
% arch fields for pkg P are set if set to anything
|
||||
arch_platform_set(P) :- arch_platform_set(P, _).
|
||||
arch_os_set(P) :- arch_os_set(P, _).
|
||||
arch_target_set(P) :- arch_target_set(P, _).
|
||||
node_platform_set(P) :- node_platform_set(P, _).
|
||||
node_os_set(P) :- node_os_set(P, _).
|
||||
|
||||
% if no platform/os is set, fall back to the defaults
|
||||
node_platform(P, A)
|
||||
:- node(P), not node_platform_set(P), node_platform_default(A).
|
||||
node_os(P, A) :- node(P), not node_os_set(P), node_os_default(A).
|
||||
|
||||
% setting os/platform on a node is a hard constraint
|
||||
node_platform(P, A) :- node(P), node_platform_set(P, A).
|
||||
node_os(P, A) :- node(P), node_os_set(P, A).
|
||||
|
||||
% avoid info warnings (see variants)
|
||||
#defined arch_platform_set/2.
|
||||
#defined arch_os_set/2.
|
||||
#defined arch_target_set/2.
|
||||
#defined node_platform_set/2.
|
||||
#defined node_os_set/2.
|
||||
|
||||
% if architecture value is set, it's the value
|
||||
arch_platform(P, A) :- node(P), arch_platform_set(P, A).
|
||||
arch_os(P, A) :- node(P), arch_os_set(P, A).
|
||||
arch_target(P, A) :- node(P), arch_target_set(P, A).
|
||||
%-----------------------------------------------------------------------------
|
||||
% Target semantics
|
||||
%-----------------------------------------------------------------------------
|
||||
% one target per node -- optimization will pick the "best" one
|
||||
1 { node_target(P, T) : target(T) } 1 :- node(P).
|
||||
|
||||
% if no architecture is set, fall back to the default architecture value.
|
||||
arch_platform(P, A) :- node(P), not arch_platform_set(P),
|
||||
arch_platform_default(A).
|
||||
arch_os(P, A) :- node(P), not arch_os_set(P), arch_os_default(A).
|
||||
arch_target(P, A) :- node(P), not arch_target_set(P), arch_target_default(A).
|
||||
% can't use targets on node if the compiler for the node doesn't support them
|
||||
:- node_target(P, T), not compiler_supports_target(C, V, T),
|
||||
node_compiler(P, C), node_compiler_version(P, C, V).
|
||||
|
||||
% propagate platform, os, target downwards
|
||||
% TODO: handle multiple dependents and arch compatibility
|
||||
arch_platform_set(D, A) :- node(D), depends_on(P, D), arch_platform_set(P, A).
|
||||
arch_os_set(D, A) :- node(D), depends_on(P, D), arch_os_set(P, A).
|
||||
arch_target_set(D, A) :- node(D), depends_on(P, D), arch_target_set(P, A).
|
||||
% if a target is set explicitly, respect it
|
||||
node_target(P, T) :- node(P), node_target_set(P, T).
|
||||
|
||||
% each node has the weight of its assigned target
|
||||
node_target_weight(P, N) :- node(P), node_target(P, T), target_weight(T, N).
|
||||
|
||||
#defined node_target_set/2.
|
||||
|
||||
%-----------------------------------------------------------------------------
|
||||
% Compiler semantics
|
||||
|
|
@ -212,14 +219,21 @@ root(D, 2) :- root(D), node(D).
|
|||
root(D, 1) :- not root(D), node(D).
|
||||
|
||||
% prefer default variants
|
||||
#minimize { N*R@5,P,V,X : variant_not_default(P, V, X, N), root(P, R) }.
|
||||
#minimize { N*R@10,P,V,X : variant_not_default(P, V, X, N), root(P, R) }.
|
||||
|
||||
% pick most preferred virtual providers
|
||||
#minimize{ N*R@4,D : provider_weight(D, N), root(P, R) }.
|
||||
#minimize{ N*R@9,D : provider_weight(D, N), root(P, R) }.
|
||||
|
||||
% prefer more recent versions.
|
||||
#minimize{ N@3,P,V : version_weight(P, V, N) }.
|
||||
#minimize{ N@8,P,V : version_weight(P, V, N) }.
|
||||
|
||||
% compiler preferences
|
||||
#maximize{ N@2,P : compiler_match(P, N) }.
|
||||
#minimize{ N@1,P : compiler_weight(P, N) }.
|
||||
#maximize{ N@7,P : compiler_match(P, N) }.
|
||||
#minimize{ N@6,P : compiler_weight(P, N) }.
|
||||
|
||||
% fastest target for node
|
||||
|
||||
% TODO: if these are slightly different by compiler (e.g., skylake is
|
||||
% best, gcc supports skylake and broadweell, clang's best is haswell)
|
||||
% things seem to get really slow.
|
||||
#minimize{ N@5,P : node_target_weight(P, N) }.
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@
|
|||
#show depends_on/3.
|
||||
#show version/2.
|
||||
#show variant_value/3.
|
||||
#show arch_platform/2.
|
||||
#show arch_os/2.
|
||||
#show arch_target/2.
|
||||
#show node_platform/2.
|
||||
#show node_os/2.
|
||||
#show node_target/2.
|
||||
#show node_compiler/2.
|
||||
#show node_compiler_version/3.
|
||||
|
|
|
|||
Loading…
Reference in a new issue