From eee502cc3b1a13fd801bb16ee8e57b2c85dd38a6 Mon Sep 17 00:00:00 2001 From: Todd Gamblin Date: Fri, 13 Apr 2018 21:39:39 -0700 Subject: [PATCH] init: Move file cache initialization out of __init__.py --- lib/spack/spack/__init__.py | 22 ------ lib/spack/spack/caches.py | 73 +++++++++++++++++++ lib/spack/spack/cmd/clean.py | 5 +- lib/spack/spack/repository.py | 19 +++-- lib/spack/spack/stage.py | 5 +- lib/spack/spack/test/cmd/clean.py | 13 ++-- lib/spack/spack/test/conftest.py | 5 +- lib/spack/spack/test/{ => util}/file_cache.py | 2 +- lib/spack/spack/{ => util}/file_cache.py | 0 9 files changed, 102 insertions(+), 42 deletions(-) create mode 100644 lib/spack/spack/caches.py rename lib/spack/spack/test/{ => util}/file_cache.py (98%) rename lib/spack/spack/{ => util}/file_cache.py (100%) diff --git a/lib/spack/spack/__init__.py b/lib/spack/spack/__init__.py index e9c4f4191c..863e8f8016 100644 --- a/lib/spack/spack/__init__.py +++ b/lib/spack/spack/__init__.py @@ -23,12 +23,9 @@ # License along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## -import os import sys import multiprocessing -from spack.paths import var_path, user_config_path - #----------------------------------------------------------------------------- # Below code imports spack packages. #----------------------------------------------------------------------------- @@ -44,20 +41,8 @@ from spack.util.path import canonicalize_path -# handle basic configuration first _config = spack.config.get_config('config') - -# Path where downloaded source code is cached -cache_path = canonicalize_path( - _config.get('source_cache', os.path.join(var_path, "cache"))) - - -# cache for miscellaneous stuff. -misc_cache_path = canonicalize_path( - _config.get('misc_cache', os.path.join(user_config_path, 'cache'))) - - # TODO: get this out of __init__.py binary_cache_retrieved_specs = set() @@ -102,13 +87,6 @@ spack_version = Version("0.11.2") -# set up the caches after getting all config options -import spack.fetch_strategy -from spack.file_cache import FileCache -misc_cache = FileCache(misc_cache_path) -fetch_cache = spack.fetch_strategy.FsCache(cache_path) - - # Set up the default packages database. import spack.error try: diff --git a/lib/spack/spack/caches.py b/lib/spack/spack/caches.py new file mode 100644 index 0000000000..ff2bb7a7c6 --- /dev/null +++ b/lib/spack/spack/caches.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://github.com/spack/spack +# Please also see the NOTICE and LICENSE files for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License (as +# published by the Free Software Foundation) version 2.1, February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +"""Caches used by Spack to store data""" +import os +import spack.paths +import spack.config +import spack.fetch_strategy +from spack.util.path import canonicalize_path +from spack.util.file_cache import FileCache + + +_misc_cache = None +_fetch_cache = None + + +def misc_cache(): + """The ``misc_cache`` is Spack's cache for small data. + + Currently the ``misc_cache`` stores indexes for virtual dependency + providers and for which packages provide which tags. + """ + global _misc_cache + + if _misc_cache is None: + config = spack.config.get_config('config') + path = config.get('misc_cache') + if not path: + path = os.path.join(spack.paths.user_config_path, 'cache') + path = canonicalize_path(path) + _misc_cache = FileCache(path) + + return _misc_cache + + +def fetch_cache(): + """Filesystem cache of downloaded archives. + + This prevents Spack from repeatedly fetch the same files when + building the same package different ways or multiple times. + """ + global _fetch_cache + + if _fetch_cache is None: + config = spack.config.get_config('config') + path = config.get('source_cache') + if not path: + path = os.path.join(spack.paths.var_path, "cache") + path = canonicalize_path(path) + _fetch_cache = spack.fetch_strategy.FsCache(path) + + return _fetch_cache diff --git a/lib/spack/spack/cmd/clean.py b/lib/spack/spack/cmd/clean.py index c072bac251..83d5116d0e 100644 --- a/lib/spack/spack/cmd/clean.py +++ b/lib/spack/spack/cmd/clean.py @@ -27,6 +27,7 @@ import llnl.util.tty as tty import spack +import spack.caches import spack.cmd description = "remove temporary build files and/or downloaded archives" @@ -81,8 +82,8 @@ def clean(parser, args): if args.downloads: tty.msg('Removing cached downloads') - spack.fetch_cache.destroy() + spack.caches.fetch_cache().destroy() if args.misc_cache: tty.msg('Removing cached information on repositories') - spack.misc_cache.destroy() + spack.caches.misc_cache().destroy() diff --git a/lib/spack/spack/repository.py b/lib/spack/spack/repository.py index 8a713b1a39..d22b5e269e 100644 --- a/lib/spack/spack/repository.py +++ b/lib/spack/spack/repository.py @@ -48,6 +48,7 @@ from llnl.util.filesystem import mkdirp, join_path, install import spack +import spack.caches import spack.error import spack.spec from spack.provider_index import ProviderIndex @@ -252,7 +253,8 @@ def make_provider_index_cache(packages_path, namespace): cache_filename = 'providers/{0}-index.yaml'.format(namespace) # Compute which packages needs to be updated in the cache - index_mtime = spack.misc_cache.mtime(cache_filename) + misc_cache = spack.caches.misc_cache() + index_mtime = misc_cache.mtime(cache_filename) needs_update = [ x for x, sinfo in fast_package_checker.items() @@ -260,19 +262,19 @@ def make_provider_index_cache(packages_path, namespace): ] # Read the old ProviderIndex, or make a new one. - index_existed = spack.misc_cache.init_entry(cache_filename) + index_existed = misc_cache.init_entry(cache_filename) if index_existed and not needs_update: # If the provider index exists and doesn't need an update # just read from it - with spack.misc_cache.read_transaction(cache_filename) as f: + with misc_cache.read_transaction(cache_filename) as f: index = ProviderIndex.from_yaml(f) else: # Otherwise we need a write transaction to update it - with spack.misc_cache.write_transaction(cache_filename) as (old, new): + with misc_cache.write_transaction(cache_filename) as (old, new): index = ProviderIndex.from_yaml(old) if old else ProviderIndex() @@ -305,7 +307,8 @@ def make_tag_index_cache(packages_path, namespace): cache_filename = 'tags/{0}-index.json'.format(namespace) # Compute which packages needs to be updated in the cache - index_mtime = spack.misc_cache.mtime(cache_filename) + misc_cache = spack.caches.misc_cache() + index_mtime = misc_cache.mtime(cache_filename) needs_update = [ x for x, sinfo in fast_package_checker.items() @@ -313,19 +316,19 @@ def make_tag_index_cache(packages_path, namespace): ] # Read the old ProviderIndex, or make a new one. - index_existed = spack.misc_cache.init_entry(cache_filename) + index_existed = misc_cache.init_entry(cache_filename) if index_existed and not needs_update: # If the provider index exists and doesn't need an update # just read from it - with spack.misc_cache.read_transaction(cache_filename) as f: + with misc_cache.read_transaction(cache_filename) as f: index = TagIndex.from_json(f) else: # Otherwise we need a write transaction to update it - with spack.misc_cache.write_transaction(cache_filename) as (old, new): + with misc_cache.write_transaction(cache_filename) as (old, new): index = TagIndex.from_json(old) if old else TagIndex() diff --git a/lib/spack/spack/stage.py b/lib/spack/spack/stage.py index d282786370..b119a874bb 100644 --- a/lib/spack/spack/stage.py +++ b/lib/spack/spack/stage.py @@ -39,6 +39,7 @@ from llnl.util.filesystem import remove_if_dead_link, remove_linked_tree import spack.paths +import spack.caches import spack.config import spack.error import spack.fetch_strategy as fs @@ -408,7 +409,7 @@ def fetch(self, mirror_only=False): url, digest, expand=expand, extension=extension)) if self.default_fetcher.cachable: fetchers.insert( - 0, spack.fetch_cache.fetcher( + 0, spack.caches.fetch_cache().fetcher( self.mirror_path, digest, expand=expand, extension=extension)) @@ -455,7 +456,7 @@ def check(self): self.fetcher.check() def cache_local(self): - spack.fetch_cache.store(self.fetcher, self.mirror_path) + spack.caches.fetch_cache().store(self.fetcher, self.mirror_path) def expand_archive(self): """Changes to the stage directory and attempt to expand the downloaded diff --git a/lib/spack/spack/test/cmd/clean.py b/lib/spack/spack/test/cmd/clean.py index 9231029c90..3bc5e4c184 100644 --- a/lib/spack/spack/test/cmd/clean.py +++ b/lib/spack/spack/test/cmd/clean.py @@ -23,7 +23,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA ############################################################################## import pytest -import spack +import spack.stage +import spack.caches import spack.main import spack.package @@ -42,8 +43,10 @@ def __call__(self, *args, **kwargs): monkeypatch.setattr(spack.package.PackageBase, 'do_clean', Counter()) monkeypatch.setattr(spack.stage, 'purge', Counter()) - monkeypatch.setattr(spack.fetch_cache, 'destroy', Counter(), raising=False) - monkeypatch.setattr(spack.misc_cache, 'destroy', Counter()) + monkeypatch.setattr( + spack.caches._fetch_cache, 'destroy', Counter(), raising=False) + monkeypatch.setattr( + spack.caches._misc_cache, 'destroy', Counter()) @pytest.mark.usefixtures( @@ -66,5 +69,5 @@ def test_function_calls(command_line, counters): # number of times assert spack.package.PackageBase.do_clean.call_count == counters[0] assert spack.stage.purge.call_count == counters[1] - assert spack.fetch_cache.destroy.call_count == counters[2] - assert spack.misc_cache.destroy.call_count == counters[3] + assert spack.caches.fetch_cache().destroy.call_count == counters[2] + assert spack.caches.misc_cache().destroy.call_count == counters[3] diff --git a/lib/spack/spack/test/conftest.py b/lib/spack/spack/test/conftest.py index 997cdb724c..6b60c31a4f 100644 --- a/lib/spack/spack/test/conftest.py +++ b/lib/spack/spack/test/conftest.py @@ -36,10 +36,11 @@ from llnl.util.filesystem import remove_linked_tree import spack -import spack.paths import spack.architecture +import spack.caches import spack.database import spack.directory_layout +import spack.paths import spack.platforms.test import spack.repository import spack.stage @@ -155,7 +156,7 @@ def fetch(self): def __str__(self): return "[mock fetch cache]" - monkeypatch.setattr(spack, 'fetch_cache', MockCache()) + monkeypatch.setattr(spack.caches, '_fetch_cache', MockCache()) # FIXME: The lines below should better be added to a fixture with diff --git a/lib/spack/spack/test/file_cache.py b/lib/spack/spack/test/util/file_cache.py similarity index 98% rename from lib/spack/spack/test/file_cache.py rename to lib/spack/spack/test/util/file_cache.py index c3c87296d2..4079a8d160 100644 --- a/lib/spack/spack/test/file_cache.py +++ b/lib/spack/spack/test/util/file_cache.py @@ -26,7 +26,7 @@ import os import pytest -from spack.file_cache import FileCache +from spack.util.file_cache import FileCache @pytest.fixture() diff --git a/lib/spack/spack/file_cache.py b/lib/spack/spack/util/file_cache.py similarity index 100% rename from lib/spack/spack/file_cache.py rename to lib/spack/spack/util/file_cache.py