hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

binary_distribution: Initialize _cached_specs at the module level and only search the mirrors in get_spec if spec is not in _cached_specs. (#14714)

Browse source 
* Initialize _cached_specs at the file level and check for spec in it before searching mirrors in try_download_spec.

* Make _cached_specs a set to avoid duplicates

* Fix packaging test

* Ignore build_cache in stage when spec.yaml files are downloaded.
This commit is contained in:
Patrick Gartung 2020-01-31 20:08:47 -06:00 committed by GitHub
parent 412c336113
commit ab36008635
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 7 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
lib/spack/spack/binary_distribution.py
View file

@ -661,7 +661,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
# Internal cache for downloaded specs # Internal cache for downloaded specs
_cached_specs = None _cached_specs = set()
def try_download_specs(urls=None, force=False): def try_download_specs(urls=None, force=False):
@ -669,7 +669,6 @@ def try_download_specs(urls=None, force=False):
Try to download the urls and cache them Try to download the urls and cache them
''' '''
global _cached_specs global _cached_specs
_cached_specs = []
if urls is None: if urls is None:
return {} return {}
for link in urls: for link in urls:
@ -687,7 +686,7 @@ def try_download_specs(urls=None, force=False):
# we need to mark this spec concrete on read-in. # we need to mark this spec concrete on read-in.
spec = Spec.from_yaml(f) spec = Spec.from_yaml(f)
spec._mark_concrete() spec._mark_concrete()
_cached_specs.append(spec) _cached_specs.add(spec)
return _cached_specs return _cached_specs
@ -701,14 +700,14 @@ def get_spec(spec=None, force=False):
if spec is None: if spec is None:
return {} return {}
specfile_name = tarball_name(spec, '.spec.yaml') specfile_name = tarball_name(spec, '.spec.yaml')
if _cached_specs:
tty.debug("Using previously-retrieved specs")
return _cached_specs
if not spack.mirror.MirrorCollection(): if not spack.mirror.MirrorCollection():
tty.debug("No Spack mirrors are currently configured") tty.debug("No Spack mirrors are currently configured")
return {} return {}
if spec in _cached_specs:
return _cached_specs
for mirror in spack.mirror.MirrorCollection().values(): for mirror in spack.mirror.MirrorCollection().values():
fetch_url_build_cache = url_util.join( fetch_url_build_cache = url_util.join(
mirror.fetch_url, _build_cache_relative_path) mirror.fetch_url, _build_cache_relative_path)
@ -732,7 +731,6 @@ def get_specs(force=False, use_arch=False, names=None):
""" """
Get spec.yaml's for build caches available on mirror Get spec.yaml's for build caches available on mirror
""" """
global _cached_specs
arch = architecture.Arch(architecture.platform(), arch = architecture.Arch(architecture.platform(),
'default_os', 'default_target') 'default_os', 'default_target')
arch_pattern = ('([^-]*-[^-]*-[^-]*)') arch_pattern = ('([^-]*-[^-]*-[^-]*)')
@ -747,10 +745,6 @@ def get_specs(force=False, use_arch=False, names=None):
names_pattern) names_pattern)
name_re = re.compile(regex_pattern) name_re = re.compile(regex_pattern)
if _cached_specs:
tty.debug("Using previously-retrieved specs")
return _cached_specs
if not spack.mirror.MirrorCollection(): if not spack.mirror.MirrorCollection():
tty.debug("No Spack mirrors are currently configured") tty.debug("No Spack mirrors are currently configured")
return {} return {}

2
lib/spack/spack/test/conftest.py
View file

@ -170,7 +170,7 @@ def ignore_stage_files():
Used to track which leftover files in the stage have been seen. Used to track which leftover files in the stage have been seen.
""" """
# to start with, ignore the .lock file at the stage root. # to start with, ignore the .lock file at the stage root.
return set(['.lock', spack.stage._source_path_subdir]) return set(['.lock', spack.stage._source_path_subdir, 'build_cache'])
def remove_whatever_it_is(path): def remove_whatever_it_is(path):

2
lib/spack/spack/test/packaging.py
View file

@ -214,7 +214,7 @@ def test_buildcache(mock_archive, tmpdir):
stage.destroy() stage.destroy()
# Remove cached binary specs since we deleted the mirror # Remove cached binary specs since we deleted the mirror
bindist._cached_specs = None bindist._cached_specs = set()
def test_relocate_text(tmpdir): def test_relocate_text(tmpdir):