repo cache: use -inf default instead of 0 (#39214)

FastPackageChecker.modified_since should use a default number < 0

When the repo cache does not exist, Spack uses mtime 0. This causes the repo
cache not to be generated when the repo has mtime 0.

Some popular package managers such as spack use 0 mtime normalization for
reproducible tarballs. So when installing spack with spack from a buildcache, the
repo cache doesn't generate

Also add some typehints
This commit is contained in:
Harmen Stoppels 2023-08-03 14:13:13 +02:00 committed by GitHub
parent be679759be
commit 6e933ac7df
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 28 deletions

View file

@ -24,7 +24,7 @@
import traceback
import types
import uuid
from typing import Dict, Union
from typing import Any, Dict, List, Union
import llnl.util.filesystem as fs
import llnl.util.lang
@ -422,7 +422,7 @@ def _create_new_cache(self) -> Dict[str, os.stat_result]:
def last_mtime(self):
return max(sinfo.st_mtime for sinfo in self._packages_to_stats.values())
def modified_since(self, since):
def modified_since(self, since: float) -> List[str]:
return [name for name, sinfo in self._packages_to_stats.items() if sinfo.st_mtime > since]
def __getitem__(self, item):
@ -548,35 +548,34 @@ class RepoIndex:
when they're needed.
``Indexers`` should be added to the ``RepoIndex`` using
``add_index(name, indexer)``, and they should support the interface
``add_indexer(name, indexer)``, and they should support the interface
defined by ``Indexer``, so that the ``RepoIndex`` can read, generate,
and update stored indices.
Generated indexes are accessed by name via ``__getitem__()``.
Generated indexes are accessed by name via ``__getitem__()``."""
"""
def __init__(self, package_checker, namespace, cache):
def __init__(
self,
package_checker: FastPackageChecker,
namespace: str,
cache: spack.util.file_cache.FileCache,
):
self.checker = package_checker
self.packages_path = self.checker.packages_path
if sys.platform == "win32":
self.packages_path = spack.util.path.convert_to_posix_path(self.packages_path)
self.namespace = namespace
self.indexers = {}
self.indexes = {}
self.indexers: Dict[str, Indexer] = {}
self.indexes: Dict[str, Any] = {}
self.cache = cache
def add_indexer(self, name, indexer):
def add_indexer(self, name: str, indexer: Indexer):
"""Add an indexer to the repo index.
Arguments:
name (str): name of this indexer
indexer (object): an object that supports create(), read(),
write(), and get_index() operations
"""
name: name of this indexer
indexer: object implementing the ``Indexer`` interface"""
self.indexers[name] = indexer
def __getitem__(self, name):
@ -597,17 +596,15 @@ def _build_all_indexes(self):
because the main bottleneck here is loading all the packages. It
can take tens of seconds to regenerate sequentially, and we'd
rather only pay that cost once rather than on several
invocations.
"""
invocations."""
for name, indexer in self.indexers.items():
self.indexes[name] = self._build_index(name, indexer)
def _build_index(self, name, indexer):
def _build_index(self, name: str, indexer: Indexer):
"""Determine which packages need an update, and update indexes."""
# Filename of the provider index cache (we assume they're all json)
cache_filename = "{0}/{1}-index.json".format(name, self.namespace)
cache_filename = f"{name}/{self.namespace}-index.json"
# Compute which packages needs to be updated in the cache
index_mtime = self.cache.mtime(cache_filename)
@ -631,8 +628,7 @@ def _build_index(self, name, indexer):
needs_update = self.checker.modified_since(new_index_mtime)
for pkg_name in needs_update:
namespaced_name = "%s.%s" % (self.namespace, pkg_name)
indexer.update(namespaced_name)
indexer.update(f"{self.namespace}.{pkg_name}")
indexer.write(new)

View file

@ -4,6 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import errno
import math
import os
import shutil
@ -151,18 +152,17 @@ def __exit__(cm, type, value, traceback):
return WriteTransaction(self._get_lock(key), acquire=WriteContextManager)
def mtime(self, key):
"""Return modification time of cache file, or 0 if it does not exist.
def mtime(self, key) -> float:
"""Return modification time of cache file, or -inf if it does not exist.
Time is in units returned by os.stat in the mtime field, which is
platform-dependent.
"""
if not self.init_entry(key):
return 0
return -math.inf
else:
sinfo = os.stat(self.cache_path(key))
return sinfo.st_mtime
return os.stat(self.cache_path(key)).st_mtime
def remove(self, key):
file = self.cache_path(key)