diff --git a/lib/spack/docs/basic_usage.rst b/lib/spack/docs/basic_usage.rst index d1f048ac05..52054a9405 100644 --- a/lib/spack/docs/basic_usage.rst +++ b/lib/spack/docs/basic_usage.rst @@ -1526,6 +1526,30 @@ any MPI implementation will do. If another package depends on error. Likewise, if you try to plug in some package that doesn't provide MPI, Spack will raise an error. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Explicit binding of virtual dependencies +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are packages that provide more than just one virtual dependency. When interacting with them, users +might want to utilize just a subset of what they could provide, and use other providers for virtuals they +need. + +It is possible to be more explicit and tell Spack which dependency should provide which virtual, using a +special syntax: + +.. code-block:: console + + $ spack spec strumpack ^[virtuals=mpi] intel-parallel-studio+mkl ^[virtuals=lapack] openblas + +Concretizing the spec above produces the following DAG: + +.. figure:: images/strumpack_virtuals.svg + :scale: 60 % + :align: center + +where ``intel-parallel-studio`` *could* provide ``mpi``, ``lapack``, and ``blas`` but is used only for the former. The ``lapack`` +and ``blas`` dependencies are satisfied by ``openblas``. + ^^^^^^^^^^^^^^^^^^^^^^^^ Specifying Specs by Hash ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/docs/images/strumpack_virtuals.svg b/lib/spack/docs/images/strumpack_virtuals.svg new file mode 100644 index 0000000000..eb580f0a58 --- /dev/null +++ b/lib/spack/docs/images/strumpack_virtuals.svg @@ -0,0 +1,534 @@ + + +G + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn + +netlib-scalapack@2.2.0%gcc@9.4.0/hkcrbrt + + + +o524gebsxavobkte3k5fglgwnedfkadf + +openblas@0.3.21%gcc@9.4.0/o524geb + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +2w3nq3n3hcj2tqlvcpewsryamltlu5tw + +intel-parallel-studio@cluster.2020.4%gcc@9.4.0/2w3nq3n + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +gguve5icmo5e4cw5o3hvvfsxremc46if + +cmake@3.25.1%gcc@9.4.0/gguve5i + + + +hkcrbrtf2qex6rvzuok5tzdrbam55pdn->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +i4avrindvhcamhurzbfdaggbj2zgsrrh + +pkgconf@1.8.0%gcc@9.4.0/i4avrin + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl + +perl@5.36.0%gcc@9.4.0/ywrpvv2 + + + +h3ujmb3ts4kxxxv77knh2knuystuerbx + +bzip2@1.0.8%gcc@9.4.0/h3ujmb3 + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->h3ujmb3ts4kxxxv77knh2knuystuerbx + + + + + + +uabgssx6lsgrevwbttslldnr5nzguprj + +gdbm@1.23%gcc@9.4.0/uabgssx + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->uabgssx6lsgrevwbttslldnr5nzguprj + + + + + + +gkw4dg2p7rdnhru3m6lcnsjbzyr7g3hb + +berkeley-db@18.1.40%gcc@9.4.0/gkw4dg2 + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->gkw4dg2p7rdnhru3m6lcnsjbzyr7g3hb + + + + + + +nizxi5u5bbrzhzwfy2qb7hatlhuswlrz + +zlib@1.2.13%gcc@9.4.0/nizxi5u + + + +ywrpvv2hgooeepdke33exkqrtdpd5gkl->nizxi5u5bbrzhzwfy2qb7hatlhuswlrz + + + + + + +idvshq5nqmygzd4uo62mdispwgxsw7id + +strumpack@7.0.1%gcc@9.4.0/idvshq5 + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->hkcrbrtf2qex6rvzuok5tzdrbam55pdn + + + +virtuals=scalapack + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh + +parmetis@4.0.3%gcc@9.4.0/imopnxj + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->imopnxjmv7cwzyiecdw2saq42qvpnauh + + + + + + +ern66gyp6qmhmpod4jaynxx4weoberfm + +metis@5.1.0%gcc@9.4.0/ern66gy + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->ern66gyp6qmhmpod4jaynxx4weoberfm + + + + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf + +butterflypack@2.2.2%gcc@9.4.0/nqiyrxl + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->nqiyrxlid6tikfpvoqdpvsjt5drs2obf + + + + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu + +slate@2022.07.00%gcc@9.4.0/4bu62ky + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->4bu62kyfuh4ikdkuyxfxjxanf7e7qopu + + + + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +7rzbmgoxhmm2jhellkgcjmn62uklf22x + +zfp@0.5.5%gcc@9.4.0/7rzbmgo + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->7rzbmgoxhmm2jhellkgcjmn62uklf22x + + + + + + +idvshq5nqmygzd4uo62mdispwgxsw7id->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +mujlx42xgttdc6u6rmiftsktpsrcmpbs + +blaspp@2022.07.00%gcc@9.4.0/mujlx42 + + + +mujlx42xgttdc6u6rmiftsktpsrcmpbs->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas + + + +mujlx42xgttdc6u6rmiftsktpsrcmpbs->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +htzjns66gmq6pjofohp26djmjnpbegho + +patchelf@0.16.1%gcc@9.4.0/htzjns6 + + + +xm3ldz3y3msfdc3hzshvxpbpg5hnt6o6 + +diffutils@3.8%gcc@9.4.0/xm3ldz3 + + + +h3ujmb3ts4kxxxv77knh2knuystuerbx->xm3ldz3y3msfdc3hzshvxpbpg5hnt6o6 + + + + + +o524gebsxavobkte3k5fglgwnedfkadf->ywrpvv2hgooeepdke33exkqrtdpd5gkl + + + + + +4vsmjofkhntilgzh4zebluqak5mdsu3x + +ca-certificates-mozilla@2023-01-10%gcc@9.4.0/4vsmjof + + + +xiro2z6na56qdd4czjhj54eag3ekbiow + +lapackpp@2022.07.00%gcc@9.4.0/xiro2z6 + + + +xiro2z6na56qdd4czjhj54eag3ekbiow->mujlx42xgttdc6u6rmiftsktpsrcmpbs + + + + + + +xiro2z6na56qdd4czjhj54eag3ekbiow->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +xiro2z6na56qdd4czjhj54eag3ekbiow->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +j5rupoqliu7kasm6xndl7ui32wgawkru + +ncurses@6.4%gcc@9.4.0/j5rupoq + + + +j5rupoqliu7kasm6xndl7ui32wgawkru->i4avrindvhcamhurzbfdaggbj2zgsrrh + + +virtuals=pkgconfig + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh->ern66gyp6qmhmpod4jaynxx4weoberfm + + + + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +imopnxjmv7cwzyiecdw2saq42qvpnauh->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +ern66gyp6qmhmpod4jaynxx4weoberfm->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->hkcrbrtf2qex6rvzuok5tzdrbam55pdn + + + +virtuals=scalapack + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +lfh3aovn65e66cs24qiehq3nd2ddojef + +arpack-ng@3.8.0%gcc@9.4.0/lfh3aov + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->lfh3aovn65e66cs24qiehq3nd2ddojef + + + + + + +57joith2sqq6sehge54vlloyolm36mdu + +sed@4.8%gcc@9.4.0/57joith + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->57joith2sqq6sehge54vlloyolm36mdu + + + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +nqiyrxlid6tikfpvoqdpvsjt5drs2obf->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +ogcucq2eod3xusvvied5ol2iobui4nsb + +libiconv@1.17%gcc@9.4.0/ogcucq2 + + + +xm3ldz3y3msfdc3hzshvxpbpg5hnt6o6->ogcucq2eod3xusvvied5ol2iobui4nsb + + + +virtuals=iconv + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->mujlx42xgttdc6u6rmiftsktpsrcmpbs + + + + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->xiro2z6na56qdd4czjhj54eag3ekbiow + + + + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +4bu62kyfuh4ikdkuyxfxjxanf7e7qopu->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + + +5xerf6imlgo4xlubacr4mljacc3edexo + +openssl@1.1.1s%gcc@9.4.0/5xerf6i + + + +5xerf6imlgo4xlubacr4mljacc3edexo->ywrpvv2hgooeepdke33exkqrtdpd5gkl + + + + + +5xerf6imlgo4xlubacr4mljacc3edexo->4vsmjofkhntilgzh4zebluqak5mdsu3x + + + + + +5xerf6imlgo4xlubacr4mljacc3edexo->nizxi5u5bbrzhzwfy2qb7hatlhuswlrz + + + + + + +v32wejd4d5lc6uka4qlrogwh5xae2h3r + +readline@8.2%gcc@9.4.0/v32wejd + + + +uabgssx6lsgrevwbttslldnr5nzguprj->v32wejd4d5lc6uka4qlrogwh5xae2h3r + + + + + + +lfh3aovn65e66cs24qiehq3nd2ddojef->o524gebsxavobkte3k5fglgwnedfkadf + + + +virtuals=blas,lapack + + + +lfh3aovn65e66cs24qiehq3nd2ddojef->2w3nq3n3hcj2tqlvcpewsryamltlu5tw + + + +virtuals=mpi + + + +lfh3aovn65e66cs24qiehq3nd2ddojef->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +2w3nq3n3hcj2tqlvcpewsryamltlu5tw->htzjns66gmq6pjofohp26djmjnpbegho + + + + + +7rzbmgoxhmm2jhellkgcjmn62uklf22x->gguve5icmo5e4cw5o3hvvfsxremc46if + + + + + +v32wejd4d5lc6uka4qlrogwh5xae2h3r->j5rupoqliu7kasm6xndl7ui32wgawkru + + + + + + +gguve5icmo5e4cw5o3hvvfsxremc46if->j5rupoqliu7kasm6xndl7ui32wgawkru + + + + + + +gguve5icmo5e4cw5o3hvvfsxremc46if->5xerf6imlgo4xlubacr4mljacc3edexo + + + + + + \ No newline at end of file diff --git a/lib/spack/docs/packaging_guide.rst b/lib/spack/docs/packaging_guide.rst index 89afac75fe..839f3b7c6f 100644 --- a/lib/spack/docs/packaging_guide.rst +++ b/lib/spack/docs/packaging_guide.rst @@ -2974,6 +2974,33 @@ The ``provides("mpi")`` call tells Spack that the ``mpich`` package can be used to satisfy the dependency of any package that ``depends_on("mpi")``. +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Providing multiple virtuals simultaneously +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Packages can provide more than one virtual dependency. Sometimes, due to implementation details, +there are subsets of those virtuals that need to be provided together by the same package. + +A well-known example is ``openblas``, which provides both the ``lapack`` and ``blas`` API in a single ``libopenblas`` +library. A package that needs ``lapack`` and ``blas`` must either use ``openblas`` to provide both, or not use +``openblas`` at all. It cannot pick one or the other. + +To express this constraint in a package, the two virtual dependencies must be listed in the same ``provides`` directive: + +.. code-block:: python + + provides('blas', 'lapack') + +This makes it impossible to select ``openblas`` as a provider for one of the two +virtual dependencies and not for the other. If you try to, Spack will report an error: + +.. code-block:: console + + $ spack spec netlib-scalapack ^[virtuals=lapack] openblas ^[virtuals=blas] atlas + ==> Error: concretization failed for the following reasons: + + 1. Package 'openblas' needs to provide both 'lapack' and 'blas' together, but provides only 'lapack' + ^^^^^^^^^^^^^^^^^^^^ Versioned Interfaces ^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/spack/spack/directives.py b/lib/spack/spack/directives.py index 7ebf68e548..bfd57fc6f9 100644 --- a/lib/spack/spack/directives.py +++ b/lib/spack/spack/directives.py @@ -573,17 +573,21 @@ def _execute_extends(pkg): return _execute_extends -@directive("provided") -def provides(*specs, **kwargs): - """Allows packages to provide a virtual dependency. If a package provides - 'mpi', other packages can declare that they depend on "mpi", and spack - can use the providing package to satisfy the dependency. +@directive(dicts=("provided", "provided_together")) +def provides(*specs, when: Optional[str] = None): + """Allows packages to provide a virtual dependency. + + If a package provides "mpi", other packages can declare that they depend on "mpi", + and spack can use the providing package to satisfy the dependency. + + Args: + *specs: virtual specs provided by this package + when: condition when this provides clause needs to be considered """ def _execute_provides(pkg): import spack.parser # Avoid circular dependency - when = kwargs.get("when") when_spec = make_when_spec(when) if not when_spec: return @@ -591,15 +595,18 @@ def _execute_provides(pkg): # ``when`` specs for ``provides()`` need a name, as they are used # to build the ProviderIndex. when_spec.name = pkg.name + spec_objs = [spack.spec.Spec(x) for x in specs] + spec_names = [x.name for x in spec_objs] + if len(spec_names) > 1: + pkg.provided_together.setdefault(when_spec, []).append(set(spec_names)) - for string in specs: - for provided_spec in spack.parser.parse(string): - if pkg.name == provided_spec.name: - raise CircularReferenceError("Package '%s' cannot provide itself." % pkg.name) + for provided_spec in spec_objs: + if pkg.name == provided_spec.name: + raise CircularReferenceError("Package '%s' cannot provide itself." % pkg.name) - if provided_spec not in pkg.provided: - pkg.provided[provided_spec] = set() - pkg.provided[provided_spec].add(when_spec) + if provided_spec not in pkg.provided: + pkg.provided[provided_spec] = set() + pkg.provided[provided_spec].add(when_spec) return _execute_provides diff --git a/lib/spack/spack/graph.py b/lib/spack/spack/graph.py index 76ebbf636e..78bf38ec0e 100644 --- a/lib/spack/spack/graph.py +++ b/lib/spack/spack/graph.py @@ -528,10 +528,15 @@ def node_entry(self, node): def edge_entry(self, edge): colormap = {"build": "dodgerblue", "link": "crimson", "run": "goldenrod"} + label = "" + if edge.virtuals: + label = f" xlabel=\"virtuals={','.join(edge.virtuals)}\"" return ( edge.parent.dag_hash(), edge.spec.dag_hash(), - f"[color=\"{':'.join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}\"]", + f"[color=\"{':'.join(colormap[x] for x in dt.flag_to_tuple(edge.depflag))}\"" + + label + + "]", ) diff --git a/lib/spack/spack/parser.py b/lib/spack/spack/parser.py index b73a189797..c69918b419 100644 --- a/lib/spack/spack/parser.py +++ b/lib/spack/spack/parser.py @@ -6,7 +6,7 @@ Here is the EBNF grammar for a spec:: - spec = [name] [node_options] { ^ node } | + spec = [name] [node_options] { ^[edge_properties] node } | [name] [node_options] hash | filename @@ -14,7 +14,8 @@ [name] [node_options] hash | filename - node_options = [@(version_list|version_pair)] [%compiler] { variant } + node_options = [@(version_list|version_pair)] [%compiler] { variant } + edge_properties = [ { bool_variant | key_value } ] hash = / id filename = (.|/|[a-zA-Z0-9-_]*/)([a-zA-Z0-9-_./]*)(.json|.yaml) @@ -64,6 +65,7 @@ from llnl.util.tty import color +import spack.deptypes import spack.error import spack.spec import spack.version @@ -126,6 +128,8 @@ class TokenType(TokenBase): """ # Dependency + START_EDGE_PROPERTIES = r"(?:\^\[)" + END_EDGE_PROPERTIES = r"(?:\])" DEPENDENCY = r"(?:\^)" # Version VERSION_HASH_PAIR = rf"(?:@(?:{GIT_VERSION_PATTERN})=(?:{VERSION}))" @@ -280,16 +284,15 @@ def next_spec( initial_spec = initial_spec or spack.spec.Spec() root_spec = SpecNodeParser(self.ctx).parse(initial_spec) while True: - if self.ctx.accept(TokenType.DEPENDENCY): - dependency = SpecNodeParser(self.ctx).parse() - - if dependency is None: - msg = ( - "this dependency sigil needs to be followed by a package name " - "or a node attribute (version, variant, etc.)" - ) - raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) + if self.ctx.accept(TokenType.START_EDGE_PROPERTIES): + edge_properties = EdgeAttributeParser(self.ctx, self.literal_str).parse() + edge_properties.setdefault("depflag", 0) + edge_properties.setdefault("virtuals", ()) + dependency = self._parse_node(root_spec) + root_spec._add_dependency(dependency, **edge_properties) + elif self.ctx.accept(TokenType.DEPENDENCY): + dependency = self._parse_node(root_spec) root_spec._add_dependency(dependency, depflag=0, virtuals=()) else: @@ -297,6 +300,18 @@ def next_spec( return root_spec + def _parse_node(self, root_spec): + dependency = SpecNodeParser(self.ctx).parse() + if dependency is None: + msg = ( + "the dependency sigil and any optional edge attributes must be followed by a " + "package name or a node attribute (version, variant, etc.)" + ) + raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) + if root_spec.concrete: + raise spack.spec.RedundantSpecError(root_spec, "^" + str(dependency)) + return dependency + def all_specs(self) -> List["spack.spec.Spec"]: """Return all the specs that remain to be parsed""" return list(iter(self.next_spec, None)) @@ -438,6 +453,41 @@ def parse(self, initial_spec: "spack.spec.Spec") -> "spack.spec.Spec": return initial_spec +class EdgeAttributeParser: + __slots__ = "ctx", "literal_str" + + def __init__(self, ctx, literal_str): + self.ctx = ctx + self.literal_str = literal_str + + def parse(self): + attributes = {} + while True: + if self.ctx.accept(TokenType.KEY_VALUE_PAIR): + name, value = self.ctx.current_token.value.split("=", maxsplit=1) + name = name.strip("'\" ") + value = value.strip("'\" ").split(",") + attributes[name] = value + if name not in ("deptypes", "virtuals"): + msg = ( + "the only edge attributes that are currently accepted " + 'are "deptypes" and "virtuals"' + ) + raise SpecParsingError(msg, self.ctx.current_token, self.literal_str) + # TODO: Add code to accept bool variants here as soon as use variants are implemented + elif self.ctx.accept(TokenType.END_EDGE_PROPERTIES): + break + else: + msg = "unexpected token in edge attributes" + raise SpecParsingError(msg, self.ctx.next_token, self.literal_str) + + # Turn deptypes=... to depflag representation + if "deptypes" in attributes: + deptype_string = attributes.pop("deptypes") + attributes["depflag"] = spack.deptypes.canonicalize(deptype_string) + return attributes + + def parse(text: str) -> List["spack.spec.Spec"]: """Parse text into a list of strings diff --git a/lib/spack/spack/provider_index.py b/lib/spack/spack/provider_index.py index 2624de56ac..32ace00a16 100644 --- a/lib/spack/spack/provider_index.py +++ b/lib/spack/spack/provider_index.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) """Classes and functions to manage providers of virtual dependencies""" -import itertools from typing import Dict, List, Optional, Set import spack.error @@ -11,33 +10,6 @@ import spack.util.spack_json as sjson -def _cross_provider_maps(lmap, rmap): - """Return a dictionary that combines constraint requests from both input. - - Args: - lmap: main provider map - rmap: provider map with additional constraints - """ - # TODO: this is pretty darned nasty, and inefficient, but there - # TODO: are not that many vdeps in most specs. - result = {} - for lspec, rspec in itertools.product(lmap, rmap): - try: - constrained = lspec.constrained(rspec) - except spack.error.UnsatisfiableSpecError: - continue - - # lp and rp are left and right provider specs. - for lp_spec, rp_spec in itertools.product(lmap[lspec], rmap[rspec]): - if lp_spec.name == rp_spec.name: - try: - const = lp_spec.constrained(rp_spec, deps=False) - result.setdefault(constrained, set()).add(const) - except spack.error.UnsatisfiableSpecError: - continue - return result - - class _IndexBase: #: This is a dict of dicts used for finding providers of particular #: virtual dependencies. The dict of dicts looks like: @@ -81,29 +53,6 @@ def providers_for(self, virtual_spec): def __contains__(self, name): return name in self.providers - def satisfies(self, other): - """Determine if the providers of virtual specs are compatible. - - Args: - other: another provider index - - Returns: - True if the providers are compatible, False otherwise. - """ - common = set(self.providers) & set(other.providers) - if not common: - return True - - # This ensures that some provider in other COULD satisfy the - # vpkg constraints on self. - result = {} - for name in common: - crossed = _cross_provider_maps(self.providers[name], other.providers[name]) - if crossed: - result[name] = crossed - - return all(c in result for c in common) - def __eq__(self, other): return self.providers == other.providers diff --git a/lib/spack/spack/solver/asp.py b/lib/spack/spack/solver/asp.py index 729a1febc4..63e32a7576 100644 --- a/lib/spack/spack/solver/asp.py +++ b/lib/spack/spack/solver/asp.py @@ -1501,6 +1501,17 @@ def package_provider_rules(self, pkg): ) self.gen.newline() + for when, sets_of_virtuals in pkg.provided_together.items(): + condition_id = self.condition( + when, name=pkg.name, msg="Virtuals are provided together" + ) + for set_id, virtuals_together in enumerate(sets_of_virtuals): + for name in virtuals_together: + self.gen.fact( + fn.pkg_fact(pkg.name, fn.provided_together(condition_id, set_id, name)) + ) + self.gen.newline() + def package_dependencies_rules(self, pkg): """Translate 'depends_on' directives into ASP logic.""" for _, conditions in sorted(pkg.dependencies.items()): @@ -1902,6 +1913,15 @@ class Body: clauses.append(fn.attr("package_hash", spec.name, spec._package_hash)) clauses.append(fn.attr("hash", spec.name, spec.dag_hash())) + edges = spec.edges_from_dependents() + virtuals = [x for x in itertools.chain.from_iterable([edge.virtuals for edge in edges])] + if not body: + for virtual in virtuals: + clauses.append(fn.attr("provider_set", spec.name, virtual)) + else: + for virtual in virtuals: + clauses.append(fn.attr("virtual_on_incoming_edges", spec.name, virtual)) + # add all clauses from dependencies if transitive: # TODO: Eventually distinguish 2 deps on the same pkg (build and link) @@ -3124,10 +3144,11 @@ def __init__(self, provided, conflicts): msg = ( "Spack concretizer internal error. Please submit a bug report and include the " "command, environment if applicable and the following error message." - f"\n {provided} is unsatisfiable, errors are:" + f"\n {provided} is unsatisfiable" ) - msg += "".join([f"\n {conflict}" for conflict in conflicts]) + if conflicts: + msg += ", errors are:" + "".join([f"\n {conflict}" for conflict in conflicts]) super(spack.error.UnsatisfiableSpecError, self).__init__(msg) diff --git a/lib/spack/spack/solver/concretize.lp b/lib/spack/spack/solver/concretize.lp index 26c7907759..3b3a547eff 100644 --- a/lib/spack/spack/solver/concretize.lp +++ b/lib/spack/spack/solver/concretize.lp @@ -113,10 +113,11 @@ unification_set(SetID, VirtualNode) multiple_nodes_attribute("node_flag_source"). multiple_nodes_attribute("depends_on"). multiple_nodes_attribute("virtual_on_edge"). +multiple_nodes_attribute("provider_set"). % Map constraint on the literal ID to facts on the node attr(Name, node(min_dupe_id, A1)) :- literal(LiteralID, Name, A1), solve_literal(LiteralID). -attr(Name, node(min_dupe_id, A1), A2) :- literal(LiteralID, Name, A1, A2), solve_literal(LiteralID). +attr(Name, node(min_dupe_id, A1), A2) :- literal(LiteralID, Name, A1, A2), solve_literal(LiteralID), not multiple_nodes_attribute(Name). attr(Name, node(min_dupe_id, A1), A2, A3) :- literal(LiteralID, Name, A1, A2, A3), solve_literal(LiteralID), not multiple_nodes_attribute(Name). attr(Name, node(min_dupe_id, A1), A2, A3, A4) :- literal(LiteralID, Name, A1, A2, A3, A4), solve_literal(LiteralID). @@ -124,6 +125,10 @@ attr(Name, node(min_dupe_id, A1), A2, A3, A4) :- literal(LiteralID, Name, A1, A2 attr("node_flag_source", node(min_dupe_id, A1), A2, node(min_dupe_id, A3)) :- literal(LiteralID, "node_flag_source", A1, A2, A3), solve_literal(LiteralID). attr("depends_on", node(min_dupe_id, A1), node(min_dupe_id, A2), A3) :- literal(LiteralID, "depends_on", A1, A2, A3), solve_literal(LiteralID). +attr("virtual_node", node(min_dupe_id, Virtual)) :- literal(LiteralID, "provider_set", _, Virtual), solve_literal(LiteralID). +attr("provider_set", node(min_dupe_id, Provider), node(min_dupe_id, Virtual)) :- literal(LiteralID, "provider_set", Provider, Virtual), solve_literal(LiteralID). +provider(node(min_dupe_id, Provider), node(min_dupe_id, Virtual)) :- literal(LiteralID, "provider_set", Provider, Virtual), solve_literal(LiteralID). + % Discriminate between "roots" that have been explicitly requested, and roots that are deduced from "virtual roots" explicitly_requested_root(node(min_dupe_id, A1)) :- literal(LiteralID, "root", A1), solve_literal(LiteralID). @@ -476,6 +481,19 @@ error(1, Msg) % Virtual dependencies %----------------------------------------------------------------------------- +% If the provider is set from the command line, its weight is 0 +possible_provider_weight(ProviderNode, VirtualNode, 0, "Set on the command line") + :- attr("provider_set", ProviderNode, VirtualNode). + +% Enforces all virtuals to be provided, if multiple of them are provided together +error(100, "Package '{0}' needs to provide both '{1}' and '{2}' together, but provides only '{1}'", Package, Virtual1, Virtual2) +:- condition_holds(ID, node(X, Package)), + pkg_fact(Package, provided_together(ID, SetID, Virtual1)), + pkg_fact(Package, provided_together(ID, SetID, Virtual2)), + Virtual1 != Virtual2, + attr("virtual_on_incoming_edges", node(X, Package), Virtual1), + not attr("virtual_on_incoming_edges", node(X, Package), Virtual2). + % if a package depends on a virtual, it's not external and we have a % provider for that virtual then it depends on the provider node_depends_on_virtual(PackageNode, Virtual, Type) @@ -494,6 +512,9 @@ attr("virtual_on_edge", PackageNode, ProviderNode, Virtual) provider(ProviderNode, node(_, Virtual)), not external(PackageNode). +attr("virtual_on_incoming_edges", ProviderNode, Virtual) + :- attr("virtual_on_edge", _, ProviderNode, Virtual). + % dependencies on virtuals also imply that the virtual is a virtual node 1 { attr("virtual_node", node(0..X-1, Virtual)) : max_dupes(Virtual, X) } :- node_depends_on_virtual(PackageNode, Virtual). @@ -501,6 +522,10 @@ attr("virtual_on_edge", PackageNode, ProviderNode, Virtual) % If there's a virtual node, we must select one and only one provider. % The provider must be selected among the possible providers. +error(100, "'{0}' cannot be a provider for the '{1}' virtual", Package, Virtual) + :- attr("provider_set", node(min_dupe_id, Package), node(min_dupe_id, Virtual)), + not virtual_condition_holds( node(min_dupe_id, Package), Virtual). + error(100, "Cannot find valid provider for virtual {0}", Virtual) :- attr("virtual_node", node(X, Virtual)), not provider(_, node(X, Virtual)). @@ -521,20 +546,6 @@ attr("root", PackageNode) :- attr("virtual_root", VirtualNode), provider(Package attr("node", PackageNode), virtual_condition_holds(PackageNode, Virtual) } 1 :- attr("virtual_node", node(X, Virtual)). -% If a spec is selected as a provider, it is for all the virtual it could provide -:- provider(PackageNode, node(X, Virtual1)), - virtual_condition_holds(PackageNode, Virtual2), - Virtual2 != Virtual1, - unification_set(SetID, PackageNode), - unification_set(SetID, node(X, Virtual2)), - not provider(PackageNode, node(X, Virtual2)). - -% If a spec is a dependency, and could provide a needed virtual, it must be a provider -:- node_depends_on_virtual(PackageNode, Virtual), - depends_on(PackageNode, PossibleProviderNode), - virtual_condition_holds(PossibleProviderNode, Virtual), - not attr("virtual_on_edge", PackageNode, PossibleProviderNode, Virtual). - % The provider provides the virtual if some provider condition holds. virtual_condition_holds(node(ProviderID, Provider), Virtual) :- virtual_condition_holds(ID, node(ProviderID, Provider), Virtual). virtual_condition_holds(ID, node(ProviderID, Provider), Virtual) :- @@ -561,6 +572,8 @@ do_not_impose(EffectID, node(X, Package)) not virtual_condition_holds(PackageNode, Virtual), internal_error("Virtual when provides not respected"). +#defined provided_together/4. + %----------------------------------------------------------------------------- % Virtual dependency weights %----------------------------------------------------------------------------- diff --git a/lib/spack/spack/spec.py b/lib/spack/spack/spec.py index 6030ff2681..20e5c3ffa3 100644 --- a/lib/spack/spack/spec.py +++ b/lib/spack/spack/spec.py @@ -59,7 +59,7 @@ import re import socket import warnings -from typing import Any, Callable, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union import llnl.path import llnl.string @@ -1464,6 +1464,26 @@ def edges_to_dependencies(self, name=None, depflag: dt.DepFlag = dt.ALL): """ return [d for d in self._dependencies.select(child=name, depflag=depflag)] + @property + def edge_attributes(self) -> str: + """Helper method to print edge attributes in spec literals""" + edges = self.edges_from_dependents() + if not edges: + return "" + + union = DependencySpec(parent=Spec(), spec=self, depflag=0, virtuals=()) + for edge in edges: + union.update_deptypes(edge.depflag) + union.update_virtuals(edge.virtuals) + deptypes_str = ( + f"deptypes={','.join(dt.flag_to_tuple(union.depflag))}" if union.depflag else "" + ) + virtuals_str = f"virtuals={','.join(union.virtuals)}" if union.virtuals else "" + if not deptypes_str and not virtuals_str: + return "" + result = f"{deptypes_str} {virtuals_str}".strip() + return f"[{result}]" + def dependencies(self, name=None, deptype: Union[dt.DepTypes, dt.DepFlag] = dt.ALL): """Return a list of direct dependencies (nodes in the DAG). @@ -3688,8 +3708,15 @@ def intersects(self, other: Union[str, "Spec"], deps: bool = True) -> bool: if other.concrete and self.concrete: return self.dag_hash() == other.dag_hash() - self_hash = self.dag_hash() if self.concrete else self.abstract_hash - other_hash = other.dag_hash() if other.concrete else other.abstract_hash + elif self.concrete: + return self.satisfies(other) + + elif other.concrete: + return other.satisfies(self) + + # From here we know both self and other are not concrete + self_hash = self.abstract_hash + other_hash = other.abstract_hash if ( self_hash @@ -3778,10 +3805,6 @@ def _intersects_dependencies(self, other): repository=spack.repo.PATH, specs=other.traverse(), restrict=True ) - # This handles cases where there are already providers for both vpkgs - if not self_index.satisfies(other_index): - return False - # These two loops handle cases where there is an overly restrictive # vpkg in one spec for a provider in the other (e.g., mpi@3: is not # compatible with mpich2) @@ -3879,7 +3902,46 @@ def satisfies(self, other: Union[str, "Spec"], deps: bool = True) -> bool: return False # If we arrived here, then rhs is abstract. At the moment we don't care about the edge - # structure of an abstract DAG - hence the deps=False parameter. + # structure of an abstract DAG, so we check if any edge could satisfy the properties + # we ask for. + lhs_edges: Dict[str, Set[DependencySpec]] = collections.defaultdict(set) + for rhs_edge in other.traverse_edges(root=False, cover="edges"): + # If we are checking for ^mpi we need to verify if there is any edge + if rhs_edge.spec.virtual: + rhs_edge.update_virtuals(virtuals=(rhs_edge.spec.name,)) + + if not rhs_edge.virtuals: + continue + + if not lhs_edges: + # Construct a map of the link/run subDAG + direct "build" edges, + # keyed by dependency name + for lhs_edge in self.traverse_edges( + root=False, cover="edges", deptype=("link", "run") + ): + lhs_edges[lhs_edge.spec.name].add(lhs_edge) + for virtual_name in lhs_edge.virtuals: + lhs_edges[virtual_name].add(lhs_edge) + + build_edges = self.edges_to_dependencies(depflag=dt.BUILD) + for lhs_edge in build_edges: + lhs_edges[lhs_edge.spec.name].add(lhs_edge) + for virtual_name in lhs_edge.virtuals: + lhs_edges[virtual_name].add(lhs_edge) + + # We don't have edges to this dependency + current_dependency_name = rhs_edge.spec.name + if current_dependency_name not in lhs_edges: + return False + + for virtual in rhs_edge.virtuals: + has_virtual = any( + virtual in edge.virtuals for edge in lhs_edges[current_dependency_name] + ) + if not has_virtual: + return False + + # Edges have been checked above already, hence deps=False return all( any(lhs.satisfies(rhs, deps=False) for lhs in self.traverse(root=False)) for rhs in other.traverse(root=False) @@ -4081,9 +4143,7 @@ def __getitem__(self, name): """ query_parameters = name.split(":") if len(query_parameters) > 2: - msg = "key has more than one ':' symbol." - msg += " At most one is admitted." - raise KeyError(msg) + raise KeyError("key has more than one ':' symbol. At most one is admitted.") name, query_parameters = query_parameters[0], query_parameters[1:] if query_parameters: @@ -4108,11 +4168,17 @@ def __getitem__(self, name): itertools.chain( # Regular specs (x for x in order() if x.name == name), + ( + x + for x in order() + if (not x.virtual) + and any(name in edge.virtuals for edge in x.edges_from_dependents()) + ), (x for x in order() if (not x.virtual) and x.package.provides(name)), ) ) except StopIteration: - raise KeyError("No spec with name %s in %s" % (name, self)) + raise KeyError(f"No spec with name {name} in {self}") if self._concrete: return SpecBuildInterface(value, name, query_parameters) @@ -4490,17 +4556,27 @@ def format_path( return str(path_ctor(*output_path_components)) def __str__(self): - sorted_nodes = [self] + sorted( + root_str = [self.format()] + sorted_dependencies = sorted( self.traverse(root=False), key=lambda x: (x.name, x.abstract_hash) ) - return " ^".join(d.format() for d in sorted_nodes).strip() + sorted_dependencies = [ + d.format("{edge_attributes} " + DEFAULT_FORMAT) for d in sorted_dependencies + ] + spec_str = " ^".join(root_str + sorted_dependencies) + return spec_str.strip() @property def colored_str(self): - sorted_nodes = [self] + sorted( + root_str = [self.cformat()] + sorted_dependencies = sorted( self.traverse(root=False), key=lambda x: (x.name, x.abstract_hash) ) - return " ^".join(d.cformat() for d in sorted_nodes).strip() + sorted_dependencies = [ + d.cformat("{edge_attributes} " + DISPLAY_FORMAT) for d in sorted_dependencies + ] + spec_str = " ^".join(root_str + sorted_dependencies) + return spec_str.strip() def install_status(self): """Helper for tree to print DB install status.""" diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index 0893b76a98..f2bf740272 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -642,3 +642,13 @@ def test_effective_deptype_run_environment(default_mock_concretization): for spec, effective_type in spack.build_environment.effective_deptypes(s, context=Context.RUN): assert effective_type & expected_flags.pop(spec.name) == effective_type assert not expected_flags, f"Missing {expected_flags.keys()} from effective_deptypes" + + +def test_monkey_patching_works_across_virtual(default_mock_concretization): + """Assert that a monkeypatched attribute is found regardless we access through the + real name or the virtual name. + """ + s = default_mock_concretization("mpileaks ^mpich") + s["mpich"].foo = "foo" + assert s["mpich"].foo == "foo" + assert s["mpi"].foo == "foo" diff --git a/lib/spack/spack/test/cmd/dependencies.py b/lib/spack/spack/test/cmd/dependencies.py index f61c19a7f1..bc615c7a3a 100644 --- a/lib/spack/spack/test/cmd/dependencies.py +++ b/lib/spack/spack/test/cmd/dependencies.py @@ -14,7 +14,14 @@ dependencies = SpackCommand("dependencies") -mpis = ["low-priority-provider", "mpich", "mpich2", "multi-provider-mpi", "zmpi"] +mpis = [ + "intel-parallel-studio", + "low-priority-provider", + "mpich", + "mpich2", + "multi-provider-mpi", + "zmpi", +] mpi_deps = ["fake"] diff --git a/lib/spack/spack/test/cmd/env.py b/lib/spack/spack/test/cmd/env.py index 24657c30f9..5d244f422c 100644 --- a/lib/spack/spack/test/cmd/env.py +++ b/lib/spack/spack/test/cmd/env.py @@ -2461,8 +2461,12 @@ def test_concretize_user_specs_together(): e.remove("mpich") e.add("mpich2") + exc_cls = spack.error.SpackError + if spack.config.get("config:concretizer") == "clingo": + exc_cls = spack.error.UnsatisfiableSpecError + # Concretizing without invalidating the concrete spec for mpileaks fails - with pytest.raises(spack.error.UnsatisfiableSpecError): + with pytest.raises(exc_cls): e.concretize() e.concretize(force=True) @@ -2494,9 +2498,12 @@ def test_duplicate_packages_raise_when_concretizing_together(): e.add("mpileaks~opt") e.add("mpich") - with pytest.raises( - spack.error.UnsatisfiableSpecError, match=r"You could consider setting `concretizer:unify`" - ): + exc_cls, match = spack.error.SpackError, None + if spack.config.get("config:concretizer") == "clingo": + exc_cls = spack.error.UnsatisfiableSpecError + match = r"You could consider setting `concretizer:unify`" + + with pytest.raises(exc_cls, match=match): e.concretize() diff --git a/lib/spack/spack/test/concretize.py b/lib/spack/spack/test/concretize.py index 04959a19b3..1dd530ac70 100644 --- a/lib/spack/spack/test/concretize.py +++ b/lib/spack/spack/test/concretize.py @@ -1838,7 +1838,8 @@ def test_installed_specs_disregard_conflicts(self, mutable_database, monkeypatch # If we concretize with --reuse it is not, since "mpich~debug" was already installed with spack.config.override("concretizer:reuse", True): s = Spec("mpich").concretized() - assert s.satisfies("~debug") + assert s.installed + assert s.satisfies("~debug"), s @pytest.mark.regression("32471") @pytest.mark.only_clingo("Use case not supported by the original concretizer") @@ -2132,14 +2133,16 @@ def test_reuse_python_from_cli_and_extension_from_db(self, mutable_database): @pytest.fixture() def duplicates_test_repository(): - builder_test_path = os.path.join(spack.paths.repos_path, "duplicates.test") - with spack.repo.use_repositories(builder_test_path) as mock_repo: + repository_path = os.path.join(spack.paths.repos_path, "duplicates.test") + with spack.repo.use_repositories(repository_path) as mock_repo: yield mock_repo @pytest.mark.usefixtures("mutable_config", "duplicates_test_repository") @pytest.mark.only_clingo("Not supported by the original concretizer") class TestConcretizeSeparately: + """Collects test on separate concretization""" + @pytest.mark.parametrize("strategy", ["minimal", "full"]) def test_two_gmake(self, strategy): """Tests that we can concretize a spec with nodes using the same build @@ -2320,3 +2323,40 @@ def test_adding_specs(self, input_specs, default_mock_concretization): assert node == container[node.dag_hash()] assert node.dag_hash() in container assert node is not container[node.dag_hash()] + + +@pytest.fixture() +def edges_test_repository(): + repository_path = os.path.join(spack.paths.repos_path, "edges.test") + with spack.repo.use_repositories(repository_path) as mock_repo: + yield mock_repo + + +@pytest.mark.usefixtures("mutable_config", "edges_test_repository") +@pytest.mark.only_clingo("Edge properties not supported by the original concretizer") +class TestConcretizeEdges: + """Collects tests on edge properties""" + + @pytest.mark.parametrize( + "spec_str,expected_satisfies,expected_not_satisfies", + [ + ("conditional-edge", ["^zlib@2.0"], ["^zlib-api"]), + ("conditional-edge~foo", ["^zlib@2.0"], ["^zlib-api"]), + ( + "conditional-edge+foo", + ["^zlib@1.0", "^zlib-api", "^[virtuals=zlib-api] zlib"], + ["^[virtuals=mpi] zlib"], + ), + ], + ) + def test_condition_triggered_by_edge_property( + self, spec_str, expected_satisfies, expected_not_satisfies + ): + """Tests that we can enforce constraints based on edge attributes""" + s = Spec(spec_str).concretized() + + for expected in expected_satisfies: + assert s.satisfies(expected), str(expected) + + for not_expected in expected_not_satisfies: + assert not s.satisfies(not_expected), str(not_expected) diff --git a/lib/spack/spack/test/package_class.py b/lib/spack/spack/test/package_class.py index d0126af230..279693a529 100644 --- a/lib/spack/spack/test/package_class.py +++ b/lib/spack/spack/test/package_class.py @@ -37,6 +37,7 @@ def mpileaks_possible_deps(mock_packages, mpi_names): "low-priority-provider": set(), "dyninst": set(["libdwarf", "libelf"]), "fake": set(), + "intel-parallel-studio": set(), "libdwarf": set(["libelf"]), "libelf": set(), "mpich": set(), diff --git a/lib/spack/spack/test/spec_dag.py b/lib/spack/spack/test/spec_dag.py index be646b1e03..3a9c0350ae 100644 --- a/lib/spack/spack/test/spec_dag.py +++ b/lib/spack/spack/test/spec_dag.py @@ -532,6 +532,7 @@ def test_normalize_mpileaks(self): assert not spec.eq_dag(expected_normalized, deptypes=True) assert not spec.eq_dag(non_unique_nodes, deptypes=True) + @pytest.mark.xfail(reason="String representation changed") def test_normalize_with_virtual_package(self): spec = Spec("mpileaks ^mpi ^libelf@1.8.11 ^libdwarf") spec.normalize() diff --git a/lib/spack/spack/test/spec_semantics.py b/lib/spack/spack/test/spec_semantics.py index 579ba4486c..87ed1e4b3f 100644 --- a/lib/spack/spack/test/spec_semantics.py +++ b/lib/spack/spack/test/spec_semantics.py @@ -294,13 +294,10 @@ def test_concrete_specs_which_satisfies_abstract(self, lhs, rhs, default_mock_co ("foo@4.0%pgi@4.5", "@1:3%pgi@4.4:4.6"), ("builtin.mock.mpich", "builtin.mpich"), ("mpileaks ^builtin.mock.mpich", "^builtin.mpich"), - ("mpileaks^mpich", "^zmpi"), - ("mpileaks^zmpi", "^mpich"), ("mpileaks^mpich@1.2", "^mpich@2.0"), ("mpileaks^mpich@4.0^callpath@1.5", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpich@2.0^callpath@1.7", "^mpich@1:3^callpath@1.4:1.6"), ("mpileaks^mpich@4.0^callpath@1.7", "^mpich@1:3^callpath@1.4:1.6"), - ("mpileaks^mpich", "^zmpi"), ("mpileaks^mpi@3", "^mpi@1.2:1.6"), ("mpileaks^mpi@3:", "^mpich2@1.4"), ("mpileaks^mpi@3:", "^mpich2"), @@ -338,30 +335,30 @@ def test_constraining_abstract_specs_with_empty_intersection(self, lhs, rhs): rhs.constrain(lhs) @pytest.mark.parametrize( - "lhs,rhs,intersection_expected", + "lhs,rhs", [ - ("mpich", "mpich +foo", True), - ("mpich", "mpich~foo", True), - ("mpich", "mpich foo=1", True), - ("mpich", "mpich++foo", True), - ("mpich", "mpich~~foo", True), - ("mpich", "mpich foo==1", True), + ("mpich", "mpich +foo"), + ("mpich", "mpich~foo"), + ("mpich", "mpich foo=1"), + ("mpich", "mpich++foo"), + ("mpich", "mpich~~foo"), + ("mpich", "mpich foo==1"), # Flags semantics is currently different from other variant - ("mpich", 'mpich cflags="-O3"', True), - ("mpich cflags=-O3", 'mpich cflags="-O3 -Ofast"', False), - ("mpich cflags=-O2", 'mpich cflags="-O3"', False), - ("multivalue-variant foo=bar", "multivalue-variant +foo", False), - ("multivalue-variant foo=bar", "multivalue-variant ~foo", False), - ("multivalue-variant fee=bar", "multivalue-variant fee=baz", False), + ("mpich", 'mpich cflags="-O3"'), + ("mpich cflags=-O3", 'mpich cflags="-O3 -Ofast"'), + ("mpich cflags=-O2", 'mpich cflags="-O3"'), + ("multivalue-variant foo=bar", "multivalue-variant +foo"), + ("multivalue-variant foo=bar", "multivalue-variant ~foo"), + ("multivalue-variant fee=bar", "multivalue-variant fee=baz"), ], ) def test_concrete_specs_which_do_not_satisfy_abstract( - self, lhs, rhs, intersection_expected, default_mock_concretization + self, lhs, rhs, default_mock_concretization ): lhs, rhs = default_mock_concretization(lhs), Spec(rhs) - assert lhs.intersects(rhs) is intersection_expected - assert rhs.intersects(lhs) is intersection_expected + assert lhs.intersects(rhs) is False + assert rhs.intersects(lhs) is False assert not lhs.satisfies(rhs) assert not rhs.satisfies(lhs) @@ -483,10 +480,14 @@ def test_intersects_virtual(self): assert Spec("mpich2").intersects(Spec("mpi")) assert Spec("zmpi").intersects(Spec("mpi")) - def test_intersects_virtual_dep_with_virtual_constraint(self): + def test_intersects_virtual_providers(self): + """Tests that we can always intersect virtual providers from abstract specs. + Concretization will give meaning to virtuals, and eventually forbid certain + configurations. + """ assert Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^openblas") - assert not Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^openblas") - assert not Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^netlib-blas") + assert Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^openblas") + assert Spec("netlib-lapack ^openblas").intersects("netlib-lapack ^netlib-blas") assert Spec("netlib-lapack ^netlib-blas").intersects("netlib-lapack ^netlib-blas") def test_intersectable_concrete_specs_must_have_the_same_hash(self): @@ -1006,6 +1007,103 @@ def test_spec_override(self): assert new_spec.compiler_flags["cflags"] == ["-O2"] assert new_spec.compiler_flags["cxxflags"] == ["-O1"] + @pytest.mark.parametrize( + "spec_str,specs_in_dag", + [ + ("hdf5 ^[virtuals=mpi] mpich", [("mpich", "mpich"), ("mpi", "mpich")]), + # Try different combinations with packages that provides a + # disjoint set of virtual dependencies + ( + "netlib-scalapack ^mpich ^openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^[virtuals=mpi] mpich ^openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^mpich ^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^[virtuals=mpi] mpich ^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "mpich"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + # Test that we can mix dependencies that provide an overlapping + # sets of virtual dependencies + ( + "netlib-scalapack ^[virtuals=mpi] intel-parallel-studio " + "^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "intel-parallel-studio"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^[virtuals=mpi] intel-parallel-studio ^openblas-with-lapack", + [ + ("mpi", "intel-parallel-studio"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + ( + "netlib-scalapack ^intel-parallel-studio ^[virtuals=lapack] openblas-with-lapack", + [ + ("mpi", "intel-parallel-studio"), + ("lapack", "openblas-with-lapack"), + ("blas", "openblas-with-lapack"), + ], + ), + # Test that we can bind more than one virtual to the same provider + ( + "netlib-scalapack ^[virtuals=lapack,blas] openblas-with-lapack", + [("lapack", "openblas-with-lapack"), ("blas", "openblas-with-lapack")], + ), + ], + ) + def test_virtual_deps_bindings(self, default_mock_concretization, spec_str, specs_in_dag): + if spack.config.get("config:concretizer") == "original": + pytest.skip("Use case not supported by the original concretizer") + + s = default_mock_concretization(spec_str) + for label, expected in specs_in_dag: + assert label in s + assert s[label].satisfies(expected), label + + @pytest.mark.parametrize( + "spec_str", + [ + # openblas-with-lapack needs to provide blas and lapack together + "netlib-scalapack ^[virtuals=blas] intel-parallel-studio ^openblas-with-lapack", + # intel-* provides blas and lapack together, openblas can provide blas only + "netlib-scalapack ^[virtuals=lapack] intel-parallel-studio ^openblas", + ], + ) + def test_unsatisfiable_virtual_deps_bindings(self, spec_str): + if spack.config.get("config:concretizer") == "original": + pytest.skip("Use case not supported by the original concretizer") + + with pytest.raises(spack.solver.asp.UnsatisfiableSpecError): + Spec(spec_str).concretized() + @pytest.mark.parametrize( "spec_str,format_str,expected", diff --git a/lib/spack/spack/test/spec_syntax.py b/lib/spack/spack/test/spec_syntax.py index 1d98731785..3cbb59e69f 100644 --- a/lib/spack/spack/test/spec_syntax.py +++ b/lib/spack/spack/test/spec_syntax.py @@ -530,6 +530,26 @@ def _specfile_for(spec_str, filename): [Token(TokenType.VERSION, value="@:0.4"), Token(TokenType.COMPILER, value="% nvhpc")], "@:0.4%nvhpc", ), + ( + "^[virtuals=mpi] openmpi", + [ + Token(TokenType.START_EDGE_PROPERTIES, value="^["), + Token(TokenType.KEY_VALUE_PAIR, value="virtuals=mpi"), + Token(TokenType.END_EDGE_PROPERTIES, value="]"), + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, value="openmpi"), + ], + "^[virtuals=mpi] openmpi", + ), + ( + "^[deptypes=link,build] zlib", + [ + Token(TokenType.START_EDGE_PROPERTIES, value="^["), + Token(TokenType.KEY_VALUE_PAIR, value="deptypes=link,build"), + Token(TokenType.END_EDGE_PROPERTIES, value="]"), + Token(TokenType.UNQUALIFIED_PACKAGE_NAME, value="zlib"), + ], + "^[deptypes=build,link] zlib", + ), ( "zlib@git.foo/bar", [ @@ -923,6 +943,9 @@ def test_disambiguate_hash_by_spec(spec1, spec2, constraint, mock_packages, monk ("x platform=test platform=test", spack.spec.DuplicateArchitectureError), ("x os=fe platform=test target=fe os=fe", spack.spec.DuplicateArchitectureError), ("x target=be platform=test os=be os=fe", spack.spec.DuplicateArchitectureError), + ("^[@foo] zlib", spack.parser.SpecParsingError), + # TODO: Remove this as soon as use variants are added and we can parse custom attributes + ("^[foo=bar] zlib", spack.parser.SpecParsingError), ], ) def test_error_conditions(text, exc_cls): diff --git a/var/spack/repos/builtin.mock/packages/intel-parallel-studio/package.py b/var/spack/repos/builtin.mock/packages/intel-parallel-studio/package.py new file mode 100644 index 0000000000..1ec5cf6932 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/intel-parallel-studio/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +class IntelParallelStudio(Package): + """Intel Parallel Studio.""" + + homepage = "https://software.intel.com/en-us/intel-parallel-studio-xe" + url = "http://tec/16225/parallel_studio_xe_2020_cluster_edition.tgz" + + version("cluster.2020.0", sha256="b1d3e3e425b2e44a06760ff173104bdf") + + provides("mpi@:3") + provides("scalapack") + provides("blas", "lapack") diff --git a/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py b/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py index 5b7bfc03c1..940dea3daf 100644 --- a/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py +++ b/var/spack/repos/builtin.mock/packages/low-priority-provider/package.py @@ -14,5 +14,5 @@ class LowPriorityProvider(Package): version("1.0", md5="0123456789abcdef0123456789abcdef") - provides("lapack") - provides("mpi") + # A low priority provider that provides both these specs together + provides("mpi", "lapack") diff --git a/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py b/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py index 070adf60bc..087cfb77cc 100644 --- a/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py +++ b/var/spack/repos/builtin.mock/packages/many-virtual-consumer/package.py @@ -19,4 +19,4 @@ class ManyVirtualConsumer(Package): # This directive is an example of imposing a constraint on a # dependency is that dependency is in the DAG. This pattern # is mainly used with virtual providers. - depends_on("low-priority-provider@1.0", when="^low-priority-provider") + depends_on("low-priority-provider@1.0", when="^[virtuals=mpi,lapack] low-priority-provider") diff --git a/var/spack/repos/builtin.mock/packages/netlib-scalapack/package.py b/var/spack/repos/builtin.mock/packages/netlib-scalapack/package.py new file mode 100644 index 0000000000..fe5d7f90a1 --- /dev/null +++ b/var/spack/repos/builtin.mock/packages/netlib-scalapack/package.py @@ -0,0 +1,20 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + + +from spack.package import * + + +class NetlibScalapack(Package): + homepage = "http://www.netlib.org/scalapack/" + url = "http://www.netlib.org/scalapack/scalapack-2.1.0.tgz" + + version("2.1.0", "b1d3e3e425b2e44a06760ff173104bdf") + + provides("scalapack") + + depends_on("mpi") + depends_on("lapack") + depends_on("blas") diff --git a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py index 0156085877..1273b70def 100644 --- a/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py +++ b/var/spack/repos/builtin.mock/packages/openblas-with-lapack/package.py @@ -14,5 +14,4 @@ class OpenblasWithLapack(Package): version("0.2.15", md5="b1190f3d3471685f17cfd1ec1d252ac9") - provides("lapack") - provides("blas") + provides("lapack", "blas") diff --git a/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py b/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py index fbcc63cded..081e07fe6c 100644 --- a/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py +++ b/var/spack/repos/builtin/packages/lua-luajit-openresty/package.py @@ -28,8 +28,7 @@ class LuaLuajitOpenresty(LuaImplPackage): description="add symlinks to make lua-luajit a drop-in lua replacement", ) - provides("lua-lang@5.1", when="+lualinks") - provides("luajit") + provides("luajit", "lua-lang@5.1", when="+lualinks") lua_version_override = "5.1" @run_after("install") diff --git a/var/spack/repos/builtin/packages/lua-luajit/package.py b/var/spack/repos/builtin/packages/lua-luajit/package.py index e8a1c124e0..dfe9f51cd0 100644 --- a/var/spack/repos/builtin/packages/lua-luajit/package.py +++ b/var/spack/repos/builtin/packages/lua-luajit/package.py @@ -33,8 +33,7 @@ class LuaLuajit(LuaImplPackage): description="add symlinks to make lua-luajit a drop-in lua replacement", ) - provides("lua-lang@5.1", when="+lualinks") - provides("luajit") + provides("luajit", "lua-lang@5.1", when="+lualinks") lua_version_override = "5.1" conflicts("platform=darwin", msg="luajit not supported on MacOS, see lua-luajit-openresty") diff --git a/var/spack/repos/builtin/packages/openblas/package.py b/var/spack/repos/builtin/packages/openblas/package.py index 43c3dafdad..409dfa004d 100644 --- a/var/spack/repos/builtin/packages/openblas/package.py +++ b/var/spack/repos/builtin/packages/openblas/package.py @@ -88,8 +88,7 @@ class Openblas(CMakePackage, MakefilePackage): ) # virtual dependency - provides("blas") - provides("lapack") + provides("blas", "lapack") provides("lapack@3.9.1:", when="@0.3.15:") provides("lapack@3.7.0", when="@0.2.20") diff --git a/var/spack/repos/edges.test/packages/conditional-edge/package.py b/var/spack/repos/edges.test/packages/conditional-edge/package.py new file mode 100644 index 0000000000..964596fcc1 --- /dev/null +++ b/var/spack/repos/edges.test/packages/conditional-edge/package.py @@ -0,0 +1,24 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class ConditionalEdge(Package): + """This package has a variant that triggers a condition only if a required dependency is + providing a virtual. + """ + + homepage = "http://www.example.com" + url = "http://www.example.com/a-1.0.tar.gz" + + version("2.0", md5="abcdef0123456789abcdef0123456789") + version("1.0", md5="0123456789abcdef0123456789abcdef") + + variant("foo", default=False, description="Just a regular foo") + + # zlib is a real package, providing zlib-api + depends_on("zlib") + depends_on("zlib-api", when="+foo") + depends_on("zlib@1.0", when="^[virtuals=zlib-api] zlib") diff --git a/var/spack/repos/edges.test/packages/zlib/package.py b/var/spack/repos/edges.test/packages/zlib/package.py new file mode 100644 index 0000000000..66dfc4f58b --- /dev/null +++ b/var/spack/repos/edges.test/packages/zlib/package.py @@ -0,0 +1,19 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) +from spack.package import * + + +class Zlib(Package): + """This package has a variant that triggers a condition only if a required dependency is + providing a virtual. + """ + + homepage = "http://www.example.com" + url = "http://www.example.com/a-1.0.tar.gz" + + version("2.0", md5="abcdef0123456789abcdef0123456789") + version("1.0", md5="0123456789abcdef0123456789abcdef") + + provides("zlib-api") diff --git a/var/spack/repos/edges.test/repo.yaml b/var/spack/repos/edges.test/repo.yaml new file mode 100644 index 0000000000..86df79affe --- /dev/null +++ b/var/spack/repos/edges.test/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: edges.test