From ab36008635e2c22e40ce608f150475f38a19b772 Mon Sep 17 00:00:00 2001 From: Patrick Gartung Date: Fri, 31 Jan 2020 20:08:47 -0600 Subject: [PATCH] binary_distribution: Initialize _cached_specs at the module level and only search the mirrors in get_spec if spec is not in _cached_specs. (#14714) * Initialize _cached_specs at the file level and check for spec in it before searching mirrors in try_download_spec. * Make _cached_specs a set to avoid duplicates * Fix packaging test * Ignore build_cache in stage when spec.yaml files are downloaded. --- lib/spack/spack/binary_distribution.py | 16 +++++----------- lib/spack/spack/test/conftest.py | 2 +- lib/spack/spack/test/packaging.py | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/lib/spack/spack/binary_distribution.py b/lib/spack/spack/binary_distribution.py index 3e5dc89313..f1834889b8 100644 --- a/lib/spack/spack/binary_distribution.py +++ b/lib/spack/spack/binary_distribution.py @@ -661,7 +661,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False, # Internal cache for downloaded specs -_cached_specs = None +_cached_specs = set() def try_download_specs(urls=None, force=False): @@ -669,7 +669,6 @@ def try_download_specs(urls=None, force=False): Try to download the urls and cache them ''' global _cached_specs - _cached_specs = [] if urls is None: return {} for link in urls: @@ -687,7 +686,7 @@ def try_download_specs(urls=None, force=False): # we need to mark this spec concrete on read-in. spec = Spec.from_yaml(f) spec._mark_concrete() - _cached_specs.append(spec) + _cached_specs.add(spec) return _cached_specs @@ -701,14 +700,14 @@ def get_spec(spec=None, force=False): if spec is None: return {} specfile_name = tarball_name(spec, '.spec.yaml') - if _cached_specs: - tty.debug("Using previously-retrieved specs") - return _cached_specs if not spack.mirror.MirrorCollection(): tty.debug("No Spack mirrors are currently configured") return {} + if spec in _cached_specs: + return _cached_specs + for mirror in spack.mirror.MirrorCollection().values(): fetch_url_build_cache = url_util.join( mirror.fetch_url, _build_cache_relative_path) @@ -732,7 +731,6 @@ def get_specs(force=False, use_arch=False, names=None): """ Get spec.yaml's for build caches available on mirror """ - global _cached_specs arch = architecture.Arch(architecture.platform(), 'default_os', 'default_target') arch_pattern = ('([^-]*-[^-]*-[^-]*)') @@ -747,10 +745,6 @@ def get_specs(force=False, use_arch=False, names=None): names_pattern) name_re = re.compile(regex_pattern) - if _cached_specs: - tty.debug("Using previously-retrieved specs") - return _cached_specs - if not spack.mirror.MirrorCollection(): tty.debug("No Spack mirrors are currently configured") return {} diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 8b8d128d2c..d4c11e1693 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -170,7 +170,7 @@ def ignore_stage_files(): Used to track which leftover files in the stage have been seen. """ # to start with, ignore the .lock file at the stage root. - return set(['.lock', spack.stage._source_path_subdir]) + return set(['.lock', spack.stage._source_path_subdir, 'build_cache']) def remove_whatever_it_is(path): diff --git a/lib/spack/spack/test/packaging.py b/lib/spack/spack/test/packaging.py index fa601196c3..edad8e29fa 100644 --- a/lib/spack/spack/test/packaging.py +++ b/lib/spack/spack/test/packaging.py @@ -214,7 +214,7 @@ def test_buildcache(mock_archive, tmpdir): stage.destroy() # Remove cached binary specs since we deleted the mirror - bindist._cached_specs = None + bindist._cached_specs = set() def test_relocate_text(tmpdir):