mirror bug fixes: symlinks, duplicate patch names, and exception handling (#13789)
* Some packages (e.g. mpfr at the time of this patch) can have patches with the same name but different contents (which apply to different versions of the package). This appends part of the patch hash to the cache file name to avoid conflicts. * Some exceptions which occur during fetching are not a subclass of SpackError and therefore do not have a 'message' attribute. This updates the logic for mirroring a single spec (add_single_spec) to produce an appropriate error message in that case (where before it failed with an AttributeError) * In various circumstances, a mirror can contain the universal storage path but not a cosmetic symlink; in this case it would not generate a symlink. Now "spack mirror create" will create a symlink for any package that doesn't have one.
This commit is contained in:
parent
fe4ccdd555
commit
c15e55c668
4 changed files with 24 additions and 17 deletions
|
@ -53,20 +53,23 @@ class MirrorCache(object):
|
||||||
def __init__(self, root):
|
def __init__(self, root):
|
||||||
self.root = os.path.abspath(root)
|
self.root = os.path.abspath(root)
|
||||||
|
|
||||||
def store(self, fetcher, relative_dest, cosmetic_path=None):
|
def store(self, fetcher, relative_dest):
|
||||||
|
"""Fetch and relocate the fetcher's target into our mirror cache."""
|
||||||
|
|
||||||
# Note this will archive package sources even if they would not
|
# Note this will archive package sources even if they would not
|
||||||
# normally be cached (e.g. the current tip of an hg/git branch)
|
# normally be cached (e.g. the current tip of an hg/git branch)
|
||||||
dst = os.path.join(self.root, relative_dest)
|
dst = os.path.join(self.root, relative_dest)
|
||||||
mkdirp(os.path.dirname(dst))
|
mkdirp(os.path.dirname(dst))
|
||||||
fetcher.archive(dst)
|
fetcher.archive(dst)
|
||||||
|
|
||||||
# Add a symlink path that a human can read to understand what resource
|
def symlink(self, mirror_ref):
|
||||||
# the archive path refers to
|
"""Symlink a human readible path in our mirror to the actual
|
||||||
if not cosmetic_path:
|
storage location."""
|
||||||
return
|
|
||||||
cosmetic_path = os.path.join(self.root, cosmetic_path)
|
cosmetic_path = os.path.join(self.root, mirror_ref.cosmetic_path)
|
||||||
relative_dst = os.path.relpath(
|
relative_dst = os.path.relpath(
|
||||||
dst, start=os.path.dirname(cosmetic_path))
|
mirror_ref.storage_path,
|
||||||
|
start=os.path.dirname(cosmetic_path))
|
||||||
if not os.path.exists(cosmetic_path):
|
if not os.path.exists(cosmetic_path):
|
||||||
mkdirp(os.path.dirname(cosmetic_path))
|
mkdirp(os.path.dirname(cosmetic_path))
|
||||||
os.symlink(relative_dst, cosmetic_path)
|
os.symlink(relative_dst, cosmetic_path)
|
||||||
|
|
|
@ -519,7 +519,7 @@ def add_single_spec(spec, mirror_root, mirror_stats):
|
||||||
else:
|
else:
|
||||||
tty.warn(
|
tty.warn(
|
||||||
"Error while fetching %s" % spec.cformat('{name}{@version}'),
|
"Error while fetching %s" % spec.cformat('{name}{@version}'),
|
||||||
exception.message)
|
getattr(exception, 'message', exception))
|
||||||
mirror_stats.error()
|
mirror_stats.error()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -199,11 +199,15 @@ def fetch(self, stage):
|
||||||
fetcher = fs.URLFetchStrategy(self.url, fetch_digest,
|
fetcher = fs.URLFetchStrategy(self.url, fetch_digest,
|
||||||
expand=bool(self.archive_sha256))
|
expand=bool(self.archive_sha256))
|
||||||
|
|
||||||
per_package_ref = os.path.join(
|
# The same package can have multiple patches with the same name but
|
||||||
self.owner.split('.')[-1], os.path.basename(self.url))
|
# with different contents, therefore apply a subset of the hash.
|
||||||
|
name = '{0}-{1}'.format(os.path.basename(self.url), fetch_digest[:7])
|
||||||
|
|
||||||
|
per_package_ref = os.path.join(self.owner.split('.')[-1], name)
|
||||||
# Reference starting with "spack." is required to avoid cyclic imports
|
# Reference starting with "spack." is required to avoid cyclic imports
|
||||||
mirror_ref = spack.mirror.mirror_archive_paths(
|
mirror_ref = spack.mirror.mirror_archive_paths(
|
||||||
fetcher, per_package_ref)
|
fetcher,
|
||||||
|
per_package_ref)
|
||||||
|
|
||||||
self.stage = spack.stage.Stage(fetcher, mirror_paths=mirror_ref)
|
self.stage = spack.stage.Stage(fetcher, mirror_paths=mirror_ref)
|
||||||
self.stage.create()
|
self.stage.create()
|
||||||
|
|
|
@ -498,13 +498,13 @@ def cache_mirror(self, stats):
|
||||||
|
|
||||||
if os.path.exists(absolute_storage_path):
|
if os.path.exists(absolute_storage_path):
|
||||||
stats.already_existed(absolute_storage_path)
|
stats.already_existed(absolute_storage_path)
|
||||||
return
|
else:
|
||||||
|
self.fetch()
|
||||||
|
spack.caches.mirror_cache.store(
|
||||||
|
self.fetcher, self.mirror_paths.storage_path)
|
||||||
|
stats.added(absolute_storage_path)
|
||||||
|
|
||||||
self.fetch()
|
spack.caches.mirror_cache.symlink(self.mirror_paths)
|
||||||
spack.caches.mirror_cache.store(
|
|
||||||
self.fetcher, self.mirror_paths.storage_path,
|
|
||||||
self.mirror_paths.cosmetic_path)
|
|
||||||
stats.added(absolute_storage_path)
|
|
||||||
|
|
||||||
def expand_archive(self):
|
def expand_archive(self):
|
||||||
"""Changes to the stage directory and attempt to expand the downloaded
|
"""Changes to the stage directory and attempt to expand the downloaded
|
||||||
|
|
Loading…
Reference in a new issue