hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

mirrors: distinguish between source/binary mirror; simplify schema (#34523)

Browse source 
Allow the following formats:

```yaml
mirrors:
  name: <url>
```

```yaml
mirrors:
  name:
    url: s3://xyz
    access_pair: [x, y]
```

```yaml
mirrors:
  name:
    fetch: http://xyz
    push:
      url: s3://xyz
      access_pair: [x, y]
```

And reserve two new properties to indicate the mirror type (e.g.
mirror.spack.io is a source mirror, not a binary cache)

```yaml
mirrors:
  spack-public:
    source: true
    binary: false
    url: https://mirror.spack.io
```
This commit is contained in:
Harmen Stoppels 2023-07-13 13:29:17 +02:00 committed by GitHub
parent 3261889e3a
commit 522d9e260b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 486 additions and 304 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
etc/spack/defaults/mirrors.yaml
View file

@ -1,2 +1,4 @@
mirrors:
spack-public: https://mirror.spack.io
spack-public:
binary: false
url: https://mirror.spack.io

31
lib/spack/spack/binary_distribution.py
View file

@ -317,9 +317,9 @@ def update(self, with_cooldown=False):
from each configured mirror and stored locally (both in memory and
on disk under ``_index_cache_root``)."""
self._init_local_index_cache()
mirrors = spack.mirror.MirrorCollection()
configured_mirror_urls = [m.fetch_url for m in mirrors.values()]
configured_mirror_urls = [
m.fetch_url for m in spack.mirror.MirrorCollection(binary=True).values()
]
items_to_remove = []
spec_cache_clear_needed = False
spec_cache_regenerate_needed = not self._mirrors_for_spec
@ -1465,8 +1465,9 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
"signature_verified": "true-if-binary-pkg-was-already-verified"
}
"""
if not spack.mirror.MirrorCollection():
tty.die("Please add a spack mirror to allow " + "download of pre-compiled packages.")
configured_mirrors = spack.mirror.MirrorCollection(binary=True).values()
if not configured_mirrors:
tty.die("Please add a spack mirror to allow download of pre-compiled packages.")
tarball = tarball_path_name(spec, ".spack")
specfile_prefix = tarball_name(spec, ".spec")
@ -1483,11 +1484,7 @@ def download_tarball(spec, unsigned=False, mirrors_for_spec=None):
# we need was in an un-indexed mirror. No need to check any
# mirror for the spec twice though.
try_first = [i["mirror_url"] for i in mirrors_for_spec] if mirrors_for_spec else []
try_next = [
i.fetch_url
for i in spack.mirror.MirrorCollection().values()
if i.fetch_url not in try_first
]
try_next = [i.fetch_url for i in configured_mirrors if i.fetch_url not in try_first]
for url in try_first + try_next:
mirrors_to_try.append(
@ -1980,7 +1977,9 @@ def try_direct_fetch(spec, mirrors=None):
specfile_is_signed = False
found_specs = []
for mirror in spack.mirror.MirrorCollection(mirrors=mirrors).values():
binary_mirrors = spack.mirror.MirrorCollection(mirrors=mirrors, binary=True).values()
for mirror in binary_mirrors:
buildcache_fetch_url_json = url_util.join(
mirror.fetch_url, _build_cache_relative_path, specfile_name
)
@ -2043,7 +2042,7 @@ def get_mirrors_for_spec(spec=None, mirrors_to_check=None, index_only=False):
if spec is None:
return []
if not spack.mirror.MirrorCollection(mirrors=mirrors_to_check):
if not spack.mirror.MirrorCollection(mirrors=mirrors_to_check, binary=True):
tty.debug("No Spack mirrors are currently configured")
return {}
@ -2082,7 +2081,7 @@ def clear_spec_cache():
def get_keys(install=False, trust=False, force=False, mirrors=None):
"""Get pgp public keys available on mirror with suffix .pub"""
mirror_collection = mirrors or spack.mirror.MirrorCollection()
mirror_collection = mirrors or spack.mirror.MirrorCollection(binary=True)
if not mirror_collection:
tty.die("Please add a spack mirror to allow " + "download of build caches.")
@ -2243,7 +2242,7 @@ def check_specs_against_mirrors(mirrors, specs, output_file=None):
"""
rebuilds = {}
for mirror in spack.mirror.MirrorCollection(mirrors).values():
for mirror in spack.mirror.MirrorCollection(mirrors, binary=True).values():
tty.debug("Checking for built specs at {0}".format(mirror.fetch_url))
rebuild_list = []
@ -2287,7 +2286,7 @@ def _download_buildcache_entry(mirror_root, descriptions):
def download_buildcache_entry(file_descriptions, mirror_url=None):
if not mirror_url and not spack.mirror.MirrorCollection():
if not mirror_url and not spack.mirror.MirrorCollection(binary=True):
tty.die(
"Please provide or add a spack mirror to allow " + "download of buildcache entries."
)
@ -2296,7 +2295,7 @@ def download_buildcache_entry(file_descriptions, mirror_url=None):
mirror_root = os.path.join(mirror_url, _build_cache_relative_path)
return _download_buildcache_entry(mirror_root, file_descriptions)
for mirror in spack.mirror.MirrorCollection().values():
for mirror in spack.mirror.MirrorCollection(binary=True).values():
mirror_root = os.path.join(mirror.fetch_url, _build_cache_relative_path)
if _download_buildcache_entry(mirror_root, file_descriptions):

2
lib/spack/spack/ci.py
View file

@ -224,7 +224,7 @@ def _print_staging_summary(spec_labels, stages, mirrors_to_check, rebuild_decisi
if not stages:
return
mirrors = spack.mirror.MirrorCollection(mirrors=mirrors_to_check)
mirrors = spack.mirror.MirrorCollection(mirrors=mirrors_to_check, binary=True)
tty.msg("Checked the following mirrors for binaries:")
for m in mirrors.values():
tty.msg(" {0}".format(m.fetch_url))

2
lib/spack/spack/cmd/gpg.py
View file

@ -216,7 +216,7 @@ def gpg_publish(args):
url = spack.util.url.path_to_file_url(args.directory)
mirror = spack.mirror.Mirror(url, url)
elif args.mirror_name:
mirror = spack.mirror.MirrorCollection().lookup(args.mirror_name)
mirror = spack.mirror.MirrorCollection(binary=True).lookup(args.mirror_name)
elif args.mirror_url:
mirror = spack.mirror.Mirror(args.mirror_url, args.mirror_url)

148
lib/spack/spack/cmd/mirror.py
View file

@ -21,7 +21,6 @@
import spack.util.path
import spack.util.web as web_util
from spack.error import SpackError
from spack.util.spack_yaml import syaml_dict
description = "manage mirrors (source and binary)"
section = "config"
@ -104,6 +103,15 @@ def setup_parser(subparser):
default=spack.config.default_modify_scope(),
help="configuration scope to modify",
)
add_parser.add_argument(
"--type",
action="append",
choices=("binary", "source"),
help=(
"specify the mirror type: for both binary "
"and source use `--type binary --type source` (default)"
),
)
arguments.add_s3_connection_args(add_parser, False)
# Remove
remove_parser = sp.add_parser("remove", aliases=["rm"], help=mirror_remove.__doc__)
@ -120,8 +128,12 @@ def setup_parser(subparser):
set_url_parser = sp.add_parser("set-url", help=mirror_set_url.__doc__)
set_url_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror")
set_url_parser.add_argument("url", help="url of mirror directory from 'spack mirror create'")
set_url_parser.add_argument(
"--push", action="store_true", help="set only the URL used for uploading new packages"
set_url_push_or_fetch = set_url_parser.add_mutually_exclusive_group(required=False)
set_url_push_or_fetch.add_argument(
"--push", action="store_true", help="set only the URL used for uploading"
)
set_url_push_or_fetch.add_argument(
"--fetch", action="store_true", help="set only the URL used for downloading"
)
set_url_parser.add_argument(
"--scope",
@ -132,6 +144,35 @@ def setup_parser(subparser):
)
arguments.add_s3_connection_args(set_url_parser, False)
# Set
set_parser = sp.add_parser("set", help=mirror_set.__doc__)
set_parser.add_argument("name", help="mnemonic name for mirror", metavar="mirror")
set_parser_push_or_fetch = set_parser.add_mutually_exclusive_group(required=False)
set_parser_push_or_fetch.add_argument(
"--push", action="store_true", help="modify just the push connection details"
)
set_parser_push_or_fetch.add_argument(
"--fetch", action="store_true", help="modify just the fetch connection details"
)
set_parser.add_argument(
"--type",
action="append",
choices=("binary", "source"),
help=(
"specify the mirror type: for both binary "
"and source use `--type binary --type source`"
),
)
set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'")
set_parser.add_argument(
"--scope",
choices=scopes,
metavar=scopes_metavar,
default=spack.config.default_modify_scope(),
help="configuration scope to modify",
)
arguments.add_s3_connection_args(set_parser, False)
# List
list_parser = sp.add_parser("list", help=mirror_list.__doc__)
list_parser.add_argument(
@ -151,17 +192,21 @@ def mirror_add(args):
or args.s3_access_token
or args.s3_profile
or args.s3_endpoint_url
or args.type
):
connection = {"url": args.url}
if args.s3_access_key_id and args.s3_access_key_secret:
connection["access_pair"] = (args.s3_access_key_id, args.s3_access_key_secret)
connection["access_pair"] = [args.s3_access_key_id, args.s3_access_key_secret]
if args.s3_access_token:
connection["access_token"] = args.s3_access_token
if args.s3_profile:
connection["profile"] = args.s3_profile
if args.s3_endpoint_url:
connection["endpoint_url"] = args.s3_endpoint_url
mirror = spack.mirror.Mirror(fetch_url=connection, push_url=connection, name=args.name)
if args.type:
connection["binary"] = "binary" in args.type
connection["source"] = "source" in args.type
mirror = spack.mirror.Mirror(connection, name=args.name)
else:
mirror = spack.mirror.Mirror(args.url, name=args.name)
spack.mirror.add(mirror, args.scope)
@ -172,75 +217,51 @@ def mirror_remove(args):
spack.mirror.remove(args.name, args.scope)
def mirror_set_url(args):
"""change the URL of a mirror"""
url = args.url
def _configure_mirror(args):
mirrors = spack.config.get("mirrors", scope=args.scope)
if not mirrors:
mirrors = syaml_dict()
if args.name not in mirrors:
tty.die("No mirror found with name %s." % args.name)
tty.die(f"No mirror found with name {args.name}.")
entry = mirrors[args.name]
key_values = ["s3_access_key_id", "s3_access_token", "s3_profile"]
entry = spack.mirror.Mirror(mirrors[args.name], args.name)
direction = "fetch" if args.fetch else "push" if args.push else None
changes = {}
if args.url:
changes["url"] = args.url
if args.s3_access_key_id and args.s3_access_key_secret:
changes["access_pair"] = [args.s3_access_key_id, args.s3_access_key_secret]
if args.s3_access_token:
changes["access_token"] = args.s3_access_token
if args.s3_profile:
changes["profile"] = args.s3_profile
if args.s3_endpoint_url:
changes["endpoint_url"] = args.s3_endpoint_url
if any(value for value in key_values if value in args):
incoming_data = {
"url": url,
"access_pair": (args.s3_access_key_id, args.s3_access_key_secret),
"access_token": args.s3_access_token,
"profile": args.s3_profile,
"endpoint_url": args.s3_endpoint_url,
}
try:
fetch_url = entry["fetch"]
push_url = entry["push"]
except TypeError:
fetch_url, push_url = entry, entry
# argparse cannot distinguish between --binary and --no-binary when same dest :(
# notice that set-url does not have these args, so getattr
if getattr(args, "type", None):
changes["binary"] = "binary" in args.type
changes["source"] = "source" in args.type
changes_made = False
changed = entry.update(changes, direction)
if args.push:
if isinstance(push_url, dict):
changes_made = changes_made or push_url != incoming_data
push_url = incoming_data
else:
changes_made = changes_made or push_url != url
push_url = url
else:
if isinstance(push_url, dict):
changes_made = changes_made or push_url != incoming_data or push_url != incoming_data
fetch_url, push_url = incoming_data, incoming_data
else:
changes_made = changes_made or push_url != url
fetch_url, push_url = url, url
items = [
(
(n, u)
if n != args.name
else (
(n, {"fetch": fetch_url, "push": push_url})
if fetch_url != push_url
else (n, {"fetch": fetch_url, "push": fetch_url})
)
)
for n, u in mirrors.items()
]
mirrors = syaml_dict(items)
spack.config.set("mirrors", mirrors, scope=args.scope)
if changes_made:
tty.msg(
"Changed%s url or connection information for mirror %s."
% ((" (push)" if args.push else ""), args.name)
)
if changed:
mirrors[args.name] = entry.to_dict()
spack.config.set("mirrors", mirrors, scope=args.scope)
else:
tty.msg("No changes made to mirror %s." % args.name)
def mirror_set(args):
"""Configure the connection details of a mirror"""
_configure_mirror(args)
def mirror_set_url(args):
"""Change the URL of a mirror."""
_configure_mirror(args)
def mirror_list(args):
"""print out available mirrors to the console"""
@ -488,6 +509,7 @@ def mirror(parser, args):
"remove": mirror_remove,
"rm": mirror_remove,
"set-url": mirror_set_url,
"set": mirror_set,
"list": mirror_list,
}

4
lib/spack/spack/installer.py
View file

@ -505,8 +505,8 @@ def _try_install_from_binary_cache(
otherwise, ``False``
timer: timer to keep track of binary install phases.
"""
# Early exit if no mirrors are configured.
if not spack.mirror.MirrorCollection():
# Early exit if no binary mirrors are configured.
if not spack.mirror.MirrorCollection(binary=True):
return False
tty.debug("Searching for binary cache of {0}".format(package_id(pkg)))

365
lib/spack/spack/mirror.py
View file

@ -18,6 +18,7 @@
import sys
import traceback
import urllib.parse
from typing import Optional, Union
import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp
@ -40,15 +41,6 @@
supported_url_schemes = ("file", "http", "https", "sftp", "ftp", "s3", "gs")
def _display_mirror_entry(size, name, url, type_=None):
if type_:
type_ = "".join((" (", type_, ")"))
else:
type_ = ""
print("%-*s%s%s" % (size + 4, name, url, type_))
def _url_or_path_to_url(url_or_path: str) -> str:
"""For simplicity we allow mirror URLs in config files to be local, relative paths.
This helper function takes care of distinguishing between URLs and paths, and
@ -71,36 +63,24 @@ class Mirror:
to them. These two URLs are usually the same.
"""
def __init__(self, fetch_url, push_url=None, name=None):
self._fetch_url = fetch_url
self._push_url = push_url
def __init__(self, data: Union[str, dict], name: Optional[str] = None):
self._data = data
self._name = name
def __eq__(self, other):
return self._fetch_url == other._fetch_url and self._push_url == other._push_url
def to_json(self, stream=None):
return sjson.dump(self.to_dict(), stream)
def to_yaml(self, stream=None):
return syaml.dump(self.to_dict(), stream)
@staticmethod
def from_yaml(stream, name=None):
data = syaml.load(stream)
return Mirror.from_dict(data, name)
return Mirror(syaml.load(stream), name)
@staticmethod
def from_json(stream, name=None):
try:
d = sjson.load(stream)
return Mirror.from_dict(d, name)
return Mirror(sjson.load(stream), name)
except Exception as e:
raise sjson.SpackJSONError("error parsing JSON mirror:", str(e)) from e
@staticmethod
def from_local_path(path: str):
return Mirror(fetch_url=url_util.path_to_file_url(path))
return Mirror(url_util.path_to_file_url(path))
@staticmethod
def from_url(url: str):
@ -111,165 +91,220 @@ def from_url(url: str):
url, ", ".join(supported_url_schemes)
)
)
return Mirror(fetch_url=url)
return Mirror(url)
def to_dict(self):
# Keep it a key-value pair <name>: <url> when possible.
if isinstance(self._fetch_url, str) and self._push_url is None:
return self._fetch_url
if self._push_url is None:
return syaml_dict([("fetch", self._fetch_url), ("push", self._fetch_url)])
else:
return syaml_dict([("fetch", self._fetch_url), ("push", self._push_url)])
@staticmethod
def from_dict(d, name=None):
if isinstance(d, str):
return Mirror(d, name=name)
else:
return Mirror(d["fetch"], d["push"], name=name)
def display(self, max_len=0):
if self._push_url is None:
_display_mirror_entry(max_len, self._name, self.fetch_url)
else:
_display_mirror_entry(max_len, self._name, self.fetch_url, "fetch")
_display_mirror_entry(max_len, self._name, self.push_url, "push")
def __eq__(self, other):
if not isinstance(other, Mirror):
return NotImplemented
return self._data == other._data and self._name == other._name
def __str__(self):
name = self._name
if name is None:
name = ""
else:
name = ' "%s"' % name
if self._push_url is None:
return "[Mirror%s (%s)]" % (name, self._fetch_url)
return "[Mirror%s (fetch: %s, push: %s)]" % (name, self._fetch_url, self._push_url)
return f"{self._name}: {self.push_url} {self.fetch_url}"
def __repr__(self):
return "".join(
(
"Mirror(",
", ".join(
"%s=%s" % (k, repr(v))
for k, v in (
("fetch_url", self._fetch_url),
("push_url", self._push_url),
("name", self._name),
)
if k == "fetch_url" or v
),
")",
)
)
return f"Mirror(name={self._name!r}, data={self._data!r})"
def to_json(self, stream=None):
return sjson.dump(self.to_dict(), stream)
def to_yaml(self, stream=None):
return syaml.dump(self.to_dict(), stream)
def to_dict(self):
return self._data
def display(self, max_len=0):
fetch, push = self.fetch_url, self.push_url
# don't print the same URL twice
url = fetch if fetch == push else f"fetch: {fetch} push: {push}"
source = "s" if self.source else " "
binary = "b" if self.binary else " "
print(f"{self.name: <{max_len}} [{source}{binary}] {url}")
@property
def name(self):
return self._name or "<unnamed>"
def get_profile(self, url_type):
if isinstance(self._fetch_url, dict):
if url_type == "push":
return self._push_url.get("profile", None)
return self._fetch_url.get("profile", None)
else:
return None
@property
def binary(self):
return isinstance(self._data, str) or self._data.get("binary", True)
def set_profile(self, url_type, profile):
if url_type == "push":
self._push_url["profile"] = profile
else:
self._fetch_url["profile"] = profile
def get_access_pair(self, url_type):
if isinstance(self._fetch_url, dict):
if url_type == "push":
return self._push_url.get("access_pair", None)
return self._fetch_url.get("access_pair", None)
else:
return None
def set_access_pair(self, url_type, connection_tuple):
if url_type == "push":
self._push_url["access_pair"] = connection_tuple
else:
self._fetch_url["access_pair"] = connection_tuple
def get_endpoint_url(self, url_type):
if isinstance(self._fetch_url, dict):
if url_type == "push":
return self._push_url.get("endpoint_url", None)
return self._fetch_url.get("endpoint_url", None)
else:
return None
def set_endpoint_url(self, url_type, url):
if url_type == "push":
self._push_url["endpoint_url"] = url
else:
self._fetch_url["endpoint_url"] = url
def get_access_token(self, url_type):
if isinstance(self._fetch_url, dict):
if url_type == "push":
return self._push_url.get("access_token", None)
return self._fetch_url.get("access_token", None)
else:
return None
def set_access_token(self, url_type, connection_token):
if url_type == "push":
self._push_url["access_token"] = connection_token
else:
self._fetch_url["access_token"] = connection_token
@property
def source(self):
return isinstance(self._data, str) or self._data.get("source", True)
@property
def fetch_url(self):
"""Get the valid, canonicalized fetch URL"""
url_or_path = (
self._fetch_url if isinstance(self._fetch_url, str) else self._fetch_url["url"]
)
return _url_or_path_to_url(url_or_path)
@fetch_url.setter
def fetch_url(self, url):
self._fetch_url["url"] = url
self._normalize()
return self.get_url("fetch")
@property
def push_url(self):
"""Get the valid, canonicalized push URL. Returns fetch URL if no custom
push URL is defined"""
if self._push_url is None:
return self.fetch_url
url_or_path = self._push_url if isinstance(self._push_url, str) else self._push_url["url"]
return _url_or_path_to_url(url_or_path)
"""Get the valid, canonicalized fetch URL"""
return self.get_url("push")
@push_url.setter
def push_url(self, url):
self._push_url["url"] = url
self._normalize()
def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool):
keys = ["url", "access_pair", "access_token", "profile", "endpoint_url"]
if top_level:
keys += ["binary", "source"]
changed = False
for key in keys:
if key in new_data and current_data.get(key) != new_data[key]:
current_data[key] = new_data[key]
changed = True
return changed
def _normalize(self):
if self._push_url is not None and self._push_url == self._fetch_url:
self._push_url = None
def update(self, data: dict, direction: Optional[str] = None) -> bool:
"""Modify the mirror with the given data. This takes care
of expanding trivial mirror definitions by URL to something more
rich with a dict if necessary
Args:
data (dict): The data to update the mirror with.
direction (str): The direction to update the mirror in (fetch
or push or None for top-level update)
Returns:
bool: True if the mirror was updated, False otherwise."""
# Modify the top-level entry when no direction is given.
if not data:
return False
# If we only update a URL, there's typically no need to expand things to a dict.
set_url = data["url"] if len(data) == 1 and "url" in data else None
if direction is None:
# First deal with the case where the current top-level entry is just a string.
if isinstance(self._data, str):
# Can we replace that string with something new?
if set_url:
if self._data == set_url:
return False
self._data = set_url
return True
# Otherwise promote to a dict
self._data = {"url": self._data}
# And update the dictionary accordingly.
return self._update_connection_dict(self._data, data, top_level=True)
# Otherwise, update the fetch / push entry; turn top-level
# url string into a dict if necessary.
if isinstance(self._data, str):
self._data = {"url": self._data}
# Create a new fetch / push entry if necessary
if direction not in self._data:
# Keep config minimal if we're just setting the URL.
if set_url:
self._data[direction] = set_url
return True
self._data[direction] = {}
entry = self._data[direction]
# Keep the entry simple if we're just swapping out the URL.
if isinstance(entry, str):
if set_url:
if entry == set_url:
return False
self._data[direction] = set_url
return True
# Otherwise promote to a dict
self._data[direction] = {"url": entry}
return self._update_connection_dict(self._data[direction], data, top_level=False)
def _get_value(self, attribute: str, direction: str):
"""Returns the most specific value for a given attribute (either push/fetch or global)"""
if direction not in ("fetch", "push"):
raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}")
if isinstance(self._data, str):
return None
# Either a string (url) or a dictionary, we care about the dict here.
value = self._data.get(direction, {})
# Return top-level entry if only a URL was set.
if isinstance(value, str):
return self._data.get(attribute, None)
return self._data.get(direction, {}).get(attribute, None)
def get_url(self, direction: str):
if direction not in ("fetch", "push"):
raise ValueError(f"direction must be either 'fetch' or 'push', not {direction}")
# Whole mirror config is just a url.
if isinstance(self._data, str):
return _url_or_path_to_url(self._data)
# Default value
url = self._data.get("url")
# Override it with a direction-specific value
if direction in self._data:
# Either a url as string or a dict with url key
info = self._data[direction]
if isinstance(info, str):
url = info
elif "url" in info:
url = info["url"]
return _url_or_path_to_url(url) if url else None
def get_access_token(self, direction: str):
return self._get_value("access_token", direction)
def get_access_pair(self, direction: str):
return self._get_value("access_pair", direction)
def get_profile(self, direction: str):
return self._get_value("profile", direction)
def get_endpoint_url(self, direction: str):
return self._get_value("endpoint_url", direction)
class MirrorCollection(collections.abc.Mapping):
"""A mapping of mirror names to mirrors."""
def __init__(self, mirrors=None, scope=None):
self._mirrors = collections.OrderedDict(
(name, Mirror.from_dict(mirror, name))
def __init__(
self,
mirrors=None,
scope=None,
binary: Optional[bool] = None,
source: Optional[bool] = None,
):
"""Initialize a mirror collection.
Args:
mirrors: A name-to-mirror mapping to initialize the collection with.
scope: The scope to use when looking up mirrors from the config.
binary: If True, only include binary mirrors.
If False, omit binary mirrors.
If None, do not filter on binary mirrors.
source: If True, only include source mirrors.
If False, omit source mirrors.
If None, do not filter on source mirrors."""
self._mirrors = {
name: Mirror(data=mirror, name=name)
for name, mirror in (
mirrors.items()
if mirrors is not None
else spack.config.get("mirrors", scope=scope).items()
)
)
}
if source is not None:
self._mirrors = {k: v for k, v in self._mirrors.items() if v.source == source}
if binary is not None:
self._mirrors = {k: v for k, v in self._mirrors.items() if v.binary == binary}
def __eq__(self, other):
return self._mirrors == other._mirrors
@ -325,7 +360,7 @@ def lookup(self, name_or_url):
result = self.get(name_or_url)
if result is None:
result = Mirror(fetch_url=name_or_url)
result = Mirror(fetch=name_or_url)
return result
@ -576,24 +611,8 @@ def remove(name, scope):
if name not in mirrors:
tty.die("No mirror with name %s" % name)
old_value = mirrors.pop(name)
mirrors.pop(name)
spack.config.set("mirrors", mirrors, scope=scope)
debug_msg_url = "url %s"
debug_msg = ["Removed mirror %s with"]
values = [name]
try:
fetch_value = old_value["fetch"]
push_value = old_value["push"]
debug_msg.extend(("fetch", debug_msg_url, "and push", debug_msg_url))
values.extend((fetch_value, push_value))
except TypeError:
debug_msg.append(debug_msg_url)
values.append(old_value)
tty.debug(" ".join(debug_msg) % tuple(values))
tty.msg("Removed mirror %s." % name)

56
lib/spack/spack/schema/mirrors.py
View file

@ -6,29 +6,55 @@
"""Schema for mirrors.yaml configuration file.
.. literalinclude:: _spack_root/lib/spack/spack/schema/mirrors.py
:lines: 12-69
"""
#: Common properties for connection specification
connection = {
"url": {"type": "string"},
# todo: replace this with named keys "username" / "password" or "id" / "secret"
"access_pair": {
"type": "array",
"items": {"type": ["string", "null"], "minItems": 2, "maxItems": 2},
},
"access_token": {"type": ["string", "null"]},
"profile": {"type": ["string", "null"]},
"endpoint_url": {"type": ["string", "null"]},
}
#: Mirror connection inside pull/push keys
fetch_and_push = {
"anyOf": [
{"type": "string"},
{
"type": "object",
"additionalProperties": False,
"properties": {**connection}, # type: ignore
},
]
}
#: Mirror connection when no pull/push keys are set
mirror_entry = {
"type": "object",
"additionalProperties": False,
"anyOf": [{"required": ["url"]}, {"required": ["fetch"]}, {"required": ["pull"]}],
"properties": {
"source": {"type": "boolean"},
"binary": {"type": "boolean"},
"fetch": fetch_and_push,
"push": fetch_and_push,
**connection, # type: ignore
},
}
#: Properties for inclusion in other schemas
properties = {
"mirrors": {
"type": "object",
"default": {},
"additionalProperties": False,
"patternProperties": {
r"\w[\w-]*": {
"anyOf": [
{"type": "string"},
{
"type": "object",
"required": ["fetch", "push"],
"properties": {
"fetch": {"type": ["string", "object"]},
"push": {"type": ["string", "object"]},
},
},
]
}
},
"patternProperties": {r"\w[\w-]*": {"anyOf": [{"type": "string"}, mirror_entry]}},
}
}

32
lib/spack/spack/stage.py
View file

@ -449,22 +449,11 @@ def fetch(self, mirror_only=False, err_msg=None):
# Join URLs of mirror roots with mirror paths. Because
# urljoin() will strip everything past the final '/' in
# the root, so we add a '/' if it is not present.
mirror_urls = {}
for mirror in spack.mirror.MirrorCollection().values():
for rel_path in self.mirror_paths:
mirror_url = url_util.join(mirror.fetch_url, rel_path)
mirror_urls[mirror_url] = {}
if (
mirror.get_access_pair("fetch")
or mirror.get_access_token("fetch")
or mirror.get_profile("fetch")
):
mirror_urls[mirror_url] = {
"access_token": mirror.get_access_token("fetch"),
"access_pair": mirror.get_access_pair("fetch"),
"access_profile": mirror.get_profile("fetch"),
"endpoint_url": mirror.get_endpoint_url("fetch"),
}
mirror_urls = [
url_util.join(mirror.fetch_url, rel_path)
for mirror in spack.mirror.MirrorCollection(source=True).values()
for rel_path in self.mirror_paths
]
# If this archive is normally fetched from a tarball URL,
# then use the same digest. `spack mirror` ensures that
@ -483,16 +472,9 @@ def fetch(self, mirror_only=False, err_msg=None):
# Add URL strategies for all the mirrors with the digest
# Insert fetchers in the order that the URLs are provided.
for url in reversed(list(mirror_urls.keys())):
for url in reversed(mirror_urls):
fetchers.insert(
0,
fs.from_url_scheme(
url,
digest,
expand=expand,
extension=extension,
connection=mirror_urls[url],
),
0, fs.from_url_scheme(url, digest, expand=expand, extension=extension)
)
if self.default_fetcher.cachable:

53
lib/spack/spack/test/cmd/mirror.py
View file

@ -149,7 +149,7 @@ def test_mirror_crud(mutable_config, capsys):
assert "No changes made" in output
output = mirror("set-url", "--push", "mirror", "s3://spack-public")
assert "Changed (push) url" in output
assert not output
# no-op
output = mirror("set-url", "--push", "mirror", "s3://spack-public")
@ -348,3 +348,54 @@ def test_versions_per_spec_produces_concrete_specs(self, input_specs, nversions,
args = MockMirrorArgs(specs=input_specs, versions_per_spec=nversions)
specs = spack.cmd.mirror.concrete_specs_from_user(args)
assert all(s.concrete for s in specs)
def test_mirror_type(mutable_config):
"""Test the mirror set command"""
mirror("add", "example", "--type", "binary", "http://example.com")
assert spack.config.get("mirrors:example") == {
"url": "http://example.com",
"source": False,
"binary": True,
}
mirror("set", "example", "--type", "source")
assert spack.config.get("mirrors:example") == {
"url": "http://example.com",
"source": True,
"binary": False,
}
mirror("set", "example", "--type", "binary")
assert spack.config.get("mirrors:example") == {
"url": "http://example.com",
"source": False,
"binary": True,
}
mirror("set", "example", "--type", "binary", "--type", "source")
assert spack.config.get("mirrors:example") == {
"url": "http://example.com",
"source": True,
"binary": True,
}
def test_mirror_set_2(mutable_config):
"""Test the mirror set command"""
mirror("add", "example", "http://example.com")
mirror(
"set",
"example",
"--push",
"--url",
"http://example2.com",
"--s3-access-key-id",
"username",
"--s3-access-key-secret",
"password",
)
assert spack.config.get("mirrors:example") == {
"url": "http://example.com",
"push": {"url": "http://example2.com", "access_pair": ["username", "password"]},
}

76
lib/spack/spack/test/mirror.py
View file

@ -132,9 +132,14 @@ def test_all_mirror(mock_git_repository, mock_svn_repository, mock_hg_repository
@pytest.mark.parametrize(
"mirror", [spack.mirror.Mirror("https://example.com/fetch", "https://example.com/push")]
"mirror",
[
spack.mirror.Mirror(
{"fetch": "https://example.com/fetch", "push": "https://example.com/push"}
)
],
)
def test_roundtrip_mirror(mirror):
def test_roundtrip_mirror(mirror: spack.mirror.Mirror):
mirror_yaml = mirror.to_yaml()
assert spack.mirror.Mirror.from_yaml(mirror_yaml) == mirror
mirror_json = mirror.to_json()
@ -291,3 +296,70 @@ def test_get_all_versions(specs, expected_specs):
output_list = [str(x) for x in output_list]
# Compare sets since order is not important
assert set(output_list) == set(expected_specs)
def test_update_1():
# No change
m = spack.mirror.Mirror("https://example.com")
assert not m.update({"url": "https://example.com"})
assert m.to_dict() == "https://example.com"
def test_update_2():
# Change URL, shouldn't expand to {"url": ...} dict.
m = spack.mirror.Mirror("https://example.com")
assert m.update({"url": "https://example.org"})
assert m.to_dict() == "https://example.org"
assert m.fetch_url == "https://example.org"
assert m.push_url == "https://example.org"
def test_update_3():
# Change fetch url, ensure minimal config
m = spack.mirror.Mirror("https://example.com")
assert m.update({"url": "https://example.org"}, "fetch")
assert m.to_dict() == {"url": "https://example.com", "fetch": "https://example.org"}
assert m.fetch_url == "https://example.org"
assert m.push_url == "https://example.com"
def test_update_4():
# Change push url, ensure minimal config
m = spack.mirror.Mirror("https://example.com")
assert m.update({"url": "https://example.org"}, "push")
assert m.to_dict() == {"url": "https://example.com", "push": "https://example.org"}
assert m.push_url == "https://example.org"
assert m.fetch_url == "https://example.com"
@pytest.mark.parametrize("direction", ["fetch", "push"])
def test_update_connection_params(direction):
"""Test whether new connection params expand the mirror config to a dict."""
m = spack.mirror.Mirror("https://example.com")
assert m.update(
{
"url": "http://example.org",
"access_pair": ["username", "password"],
"access_token": "token",
"profile": "profile",
"endpoint_url": "https://example.com",
},
direction,
)
assert m.to_dict() == {
"url": "https://example.com",
direction: {
"url": "http://example.org",
"access_pair": ["username", "password"],
"access_token": "token",
"profile": "profile",
"endpoint_url": "https://example.com",
},
}
assert m.get_access_pair(direction) == ["username", "password"]
assert m.get_access_token(direction) == "token"
assert m.get_profile(direction) == "profile"
assert m.get_endpoint_url(direction) == "https://example.com"

2
lib/spack/spack/test/web.py
View file

@ -267,7 +267,7 @@ def head_object(self, Bucket=None, Key=None):
def test_gather_s3_information(monkeypatch, capfd):
mirror = spack.mirror.Mirror.from_dict(
mirror = spack.mirror.Mirror(
{
"fetch": {
"access_token": "AAAAAAA",

15
share/spack/spack-completion.bash
View file

@ -1290,7 +1290,7 @@ _spack_mirror() {
then
SPACK_COMPREPLY="-h --help -n --no-checksum --deprecated"
else
SPACK_COMPREPLY="create destroy add remove rm set-url list"
SPACK_COMPREPLY="create destroy add remove rm set-url set list"
fi
}
@ -1310,7 +1310,7 @@ _spack_mirror_destroy() {
_spack_mirror_add() {
if $list_options
then
SPACK_COMPREPLY="-h --help --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
SPACK_COMPREPLY="-h --help --scope --type --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
else
_mirrors
fi
@ -1337,7 +1337,16 @@ _spack_mirror_rm() {
_spack_mirror_set_url() {
if $list_options
then
SPACK_COMPREPLY="-h --help --push --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
SPACK_COMPREPLY="-h --help --push --fetch --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
else
_mirrors
fi
}
_spack_mirror_set() {
if $list_options
then
SPACK_COMPREPLY="-h --help --push --fetch --type --url --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url"
else
_mirrors
fi