Allow packages to be pushed to build cache after install from source (#42423)
This commit adds a property `autopush` to mirrors. When true, every source build is immediately followed by a push to the build cache. This is useful in ephemeral environments such as CI / containers. To enable autopush on existing build caches, use `spack mirror set --autopush <name>`. The same flag can be used in `spack mirror add`.
This commit is contained in:
Radim Janalík
GitHub
parent
b76e9a887b
commit
d23e06c27e
9 changed files with 167 additions and 18 deletions
|
|
@ -220,6 +220,40 @@ section of the configuration:
|
|||
|
||||
.. _binary_caches_oci:
|
||||
|
||||
---------------------------------
|
||||
Automatic push to a build cache
|
||||
---------------------------------
|
||||
|
||||
Sometimes it is convenient to push packages to a build cache as soon as they are installed. Spack can do this by setting autopush flag when adding a mirror:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ spack mirror add --autopush <name> <url or path>
|
||||
|
||||
Or the autopush flag can be set for an existing mirror:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ spack mirror set --autopush <name> # enable automatic push for an existing mirror
|
||||
$ spack mirror set --no-autopush <name> # disable automatic push for an existing mirror
|
||||
|
||||
Then after installing a package it is automatically pushed to all mirrors with ``autopush: true``. The command
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ spack install <package>
|
||||
|
||||
will have the same effect as
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ spack install <package>
|
||||
$ spack buildcache push <cache> <package> # for all caches with autopush: true
|
||||
|
||||
.. note::
|
||||
|
||||
Packages are automatically pushed to a build cache only if they are built from source.
|
||||
|
||||
-----------------------------------------
|
||||
OCI / Docker V2 registries as build cache
|
||||
-----------------------------------------
|
||||
|
|
|
|||
|
|
@ -108,6 +108,11 @@ def setup_parser(subparser):
|
|||
"and source use `--type binary --type source` (default)"
|
||||
),
|
||||
)
|
||||
add_parser.add_argument(
|
||||
"--autopush",
|
||||
action="store_true",
|
||||
help=("set mirror to push automatically after installation"),
|
||||
)
|
||||
add_parser_signed = add_parser.add_mutually_exclusive_group(required=False)
|
||||
add_parser_signed.add_argument(
|
||||
"--unsigned",
|
||||
|
|
@ -175,6 +180,21 @@ def setup_parser(subparser):
|
|||
),
|
||||
)
|
||||
set_parser.add_argument("--url", help="url of mirror directory from 'spack mirror create'")
|
||||
set_parser_autopush = set_parser.add_mutually_exclusive_group(required=False)
|
||||
set_parser_autopush.add_argument(
|
||||
"--autopush",
|
||||
help="set mirror to push automatically after installation",
|
||||
action="store_true",
|
||||
default=None,
|
||||
dest="autopush",
|
||||
)
|
||||
set_parser_autopush.add_argument(
|
||||
"--no-autopush",
|
||||
help="set mirror to not push automatically after installation",
|
||||
action="store_false",
|
||||
default=None,
|
||||
dest="autopush",
|
||||
)
|
||||
set_parser_unsigned = set_parser.add_mutually_exclusive_group(required=False)
|
||||
set_parser_unsigned.add_argument(
|
||||
"--unsigned",
|
||||
|
|
@ -218,6 +238,7 @@ def mirror_add(args):
|
|||
or args.type
|
||||
or args.oci_username
|
||||
or args.oci_password
|
||||
or args.autopush
|
||||
or args.signed is not None
|
||||
):
|
||||
connection = {"url": args.url}
|
||||
|
|
@ -234,6 +255,8 @@ def mirror_add(args):
|
|||
if args.type:
|
||||
connection["binary"] = "binary" in args.type
|
||||
connection["source"] = "source" in args.type
|
||||
if args.autopush:
|
||||
connection["autopush"] = args.autopush
|
||||
if args.signed is not None:
|
||||
connection["signed"] = args.signed
|
||||
mirror = spack.mirror.Mirror(connection, name=args.name)
|
||||
|
|
@ -270,6 +293,8 @@ def _configure_mirror(args):
|
|||
changes["access_pair"] = [args.oci_username, args.oci_password]
|
||||
if getattr(args, "signed", None) is not None:
|
||||
changes["signed"] = args.signed
|
||||
if getattr(args, "autopush", None) is not None:
|
||||
changes["autopush"] = args.autopush
|
||||
|
||||
# argparse cannot distinguish between --binary and --no-binary when same dest :(
|
||||
# notice that set-url does not have these args, so getattr
|
||||
|
|
|
|||
27
lib/spack/spack/hooks/autopush.py
Normal file
27
lib/spack/spack/hooks/autopush.py
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import llnl.util.tty as tty
|
||||
|
||||
import spack.binary_distribution as bindist
|
||||
import spack.mirror
|
||||
|
||||
|
||||
def post_install(spec, explicit):
|
||||
# Push package to all buildcaches with autopush==True
|
||||
|
||||
# Do nothing if package was not installed from source
|
||||
pkg = spec.package
|
||||
if pkg.installed_from_binary_cache:
|
||||
return
|
||||
|
||||
# Push the package to all autopush mirrors
|
||||
for mirror in spack.mirror.MirrorCollection(binary=True, autopush=True).values():
|
||||
bindist.push_or_raise(
|
||||
spec,
|
||||
mirror.push_url,
|
||||
bindist.PushOptions(force=True, regenerate_index=False, unsigned=not mirror.signed),
|
||||
)
|
||||
tty.msg(f"{spec.name}: Pushed to build cache: '{mirror.name}'")
|
||||
|
|
@ -137,6 +137,12 @@ def source(self):
|
|||
def signed(self) -> bool:
|
||||
return isinstance(self._data, str) or self._data.get("signed", True)
|
||||
|
||||
@property
|
||||
def autopush(self) -> bool:
|
||||
if isinstance(self._data, str):
|
||||
return False
|
||||
return self._data.get("autopush", False)
|
||||
|
||||
@property
|
||||
def fetch_url(self):
|
||||
"""Get the valid, canonicalized fetch URL"""
|
||||
|
|
@ -150,7 +156,7 @@ def push_url(self):
|
|||
def _update_connection_dict(self, current_data: dict, new_data: dict, top_level: bool):
|
||||
keys = ["url", "access_pair", "access_token", "profile", "endpoint_url"]
|
||||
if top_level:
|
||||
keys += ["binary", "source", "signed"]
|
||||
keys += ["binary", "source", "signed", "autopush"]
|
||||
changed = False
|
||||
for key in keys:
|
||||
if key in new_data and current_data.get(key) != new_data[key]:
|
||||
|
|
@ -286,6 +292,7 @@ def __init__(
|
|||
scope=None,
|
||||
binary: Optional[bool] = None,
|
||||
source: Optional[bool] = None,
|
||||
autopush: Optional[bool] = None,
|
||||
):
|
||||
"""Initialize a mirror collection.
|
||||
|
||||
|
|
@ -297,21 +304,27 @@ def __init__(
|
|||
If None, do not filter on binary mirrors.
|
||||
source: If True, only include source mirrors.
|
||||
If False, omit source mirrors.
|
||||
If None, do not filter on source mirrors."""
|
||||
self._mirrors = {
|
||||
name: Mirror(data=mirror, name=name)
|
||||
for name, mirror in (
|
||||
mirrors.items()
|
||||
if mirrors is not None
|
||||
else spack.config.get("mirrors", scope=scope).items()
|
||||
)
|
||||
}
|
||||
If None, do not filter on source mirrors.
|
||||
autopush: If True, only include mirrors that have autopush enabled.
|
||||
If False, omit mirrors that have autopush enabled.
|
||||
If None, do not filter on autopush."""
|
||||
mirrors_data = (
|
||||
mirrors.items()
|
||||
if mirrors is not None
|
||||
else spack.config.get("mirrors", scope=scope).items()
|
||||
)
|
||||
mirrors = (Mirror(data=mirror, name=name) for name, mirror in mirrors_data)
|
||||
|
||||
if source is not None:
|
||||
self._mirrors = {k: v for k, v in self._mirrors.items() if v.source == source}
|
||||
def _filter(m: Mirror):
|
||||
if source is not None and m.source != source:
|
||||
return False
|
||||
if binary is not None and m.binary != binary:
|
||||
return False
|
||||
if autopush is not None and m.autopush != autopush:
|
||||
return False
|
||||
return True
|
||||
|
||||
if binary is not None:
|
||||
self._mirrors = {k: v for k, v in self._mirrors.items() if v.binary == binary}
|
||||
self._mirrors = {m.name: m for m in mirrors if _filter(m)}
|
||||
|
||||
def __eq__(self, other):
|
||||
return self._mirrors == other._mirrors
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@
|
|||
"signed": {"type": "boolean"},
|
||||
"fetch": fetch_and_push,
|
||||
"push": fetch_and_push,
|
||||
"autopush": {"type": "boolean"},
|
||||
**connection, # type: ignore
|
||||
},
|
||||
}
|
||||
|
|
|
|||
|
|
@ -169,6 +169,25 @@ def test_update_key_index(
|
|||
assert "index.json" in key_dir_list
|
||||
|
||||
|
||||
def test_buildcache_autopush(tmp_path, install_mockery, mock_fetch):
|
||||
"""Test buildcache with autopush"""
|
||||
mirror_dir = tmp_path / "mirror"
|
||||
mirror_autopush_dir = tmp_path / "mirror_autopush"
|
||||
|
||||
mirror("add", "--unsigned", "mirror", mirror_dir.as_uri())
|
||||
mirror("add", "--autopush", "--unsigned", "mirror-autopush", mirror_autopush_dir.as_uri())
|
||||
|
||||
s = Spec("libdwarf").concretized()
|
||||
|
||||
# Install and generate build cache index
|
||||
s.package.do_install()
|
||||
|
||||
metadata_file = spack.binary_distribution.tarball_name(s, ".spec.json")
|
||||
|
||||
assert not (mirror_dir / "build_cache" / metadata_file).exists()
|
||||
assert (mirror_autopush_dir / "build_cache" / metadata_file).exists()
|
||||
|
||||
|
||||
def test_buildcache_sync(
|
||||
mutable_mock_env_path,
|
||||
install_mockery_mutable_config,
|
||||
|
|
|
|||
|
|
@ -407,3 +407,27 @@ def test_mirror_add_set_signed(mutable_config):
|
|||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": False}
|
||||
mirror("set", "--signed", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "signed": True}
|
||||
|
||||
|
||||
def test_mirror_add_set_autopush(mutable_config):
|
||||
# Add mirror without autopush
|
||||
mirror("add", "example", "http://example.com")
|
||||
assert spack.config.get("mirrors:example") == "http://example.com"
|
||||
mirror("set", "--no-autopush", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
|
||||
mirror("set", "--autopush", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
|
||||
mirror("set", "--no-autopush", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
|
||||
mirror("remove", "example")
|
||||
|
||||
# Add mirror with autopush
|
||||
mirror("add", "--autopush", "example", "http://example.com")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
|
||||
mirror("set", "--autopush", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
|
||||
mirror("set", "--no-autopush", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": False}
|
||||
mirror("set", "--autopush", "example")
|
||||
assert spack.config.get("mirrors:example") == {"url": "http://example.com", "autopush": True}
|
||||
mirror("remove", "example")
|
||||
|
|
|
|||
|
|
@ -1441,7 +1441,7 @@ _spack_mirror_destroy() {
|
|||
_spack_mirror_add() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help --scope --type --unsigned --signed --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
|
||||
SPACK_COMPREPLY="-h --help --scope --type --autopush --unsigned --signed --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
|
||||
else
|
||||
_mirrors
|
||||
fi
|
||||
|
|
@ -1477,7 +1477,7 @@ _spack_mirror_set_url() {
|
|||
_spack_mirror_set() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help --push --fetch --type --url --unsigned --signed --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
|
||||
SPACK_COMPREPLY="-h --help --push --fetch --type --url --autopush --no-autopush --unsigned --signed --scope --s3-access-key-id --s3-access-key-secret --s3-access-token --s3-profile --s3-endpoint-url --oci-username --oci-password"
|
||||
else
|
||||
_mirrors
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -2253,7 +2253,7 @@ complete -c spack -n '__fish_spack_using_command mirror destroy' -l mirror-url -
|
|||
complete -c spack -n '__fish_spack_using_command mirror destroy' -l mirror-url -r -d 'find mirror to destroy by url'
|
||||
|
||||
# spack mirror add
|
||||
set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= unsigned signed s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
|
||||
set -g __fish_spack_optspecs_spack_mirror_add h/help scope= type= autopush unsigned signed s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 mirror add' -f
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -f -a help
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -s h -l help -d 'show this help message and exit'
|
||||
|
|
@ -2261,6 +2261,8 @@ complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -f -a '
|
|||
complete -c spack -n '__fish_spack_using_command mirror add' -l scope -r -d 'configuration scope to modify'
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -f -a 'binary source'
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source` (default)'
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l autopush -f -a autopush
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l autopush -d 'set mirror to push automatically after installation'
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l unsigned -f -a signed
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache'
|
||||
complete -c spack -n '__fish_spack_using_command mirror add' -l signed -f -a signed
|
||||
|
|
@ -2323,7 +2325,7 @@ complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-password
|
|||
complete -c spack -n '__fish_spack_using_command mirror set-url' -l oci-password -r -d 'password to use to connect to this OCI mirror'
|
||||
|
||||
# spack mirror set
|
||||
set -g __fish_spack_optspecs_spack_mirror_set h/help push fetch type= url= unsigned signed scope= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
|
||||
set -g __fish_spack_optspecs_spack_mirror_set h/help push fetch type= url= autopush no-autopush unsigned signed scope= s3-access-key-id= s3-access-key-secret= s3-access-token= s3-profile= s3-endpoint-url= oci-username= oci-password=
|
||||
complete -c spack -n '__fish_spack_using_command_pos 0 mirror set' -f -a '(__fish_spack_mirrors)'
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -s h -l help -f -a help
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -s h -l help -d 'show this help message and exit'
|
||||
|
|
@ -2335,6 +2337,10 @@ complete -c spack -n '__fish_spack_using_command mirror set' -l type -r -f -a 'b
|
|||
complete -c spack -n '__fish_spack_using_command mirror set' -l type -r -d 'specify the mirror type: for both binary and source use `--type binary --type source`'
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l url -r -f -a url
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l url -r -d 'url of mirror directory from \'spack mirror create\''
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l autopush -f -a autopush
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l autopush -d 'set mirror to push automatically after installation'
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l no-autopush -f -a autopush
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l no-autopush -d 'set mirror to not push automatically after installation'
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -f -a signed
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l unsigned -d 'do not require signing and signature verification when pushing and installing from this build cache'
|
||||
complete -c spack -n '__fish_spack_using_command mirror set' -l signed -f -a signed
|
||||
|
|
|
|||
Loading…
Reference in a new issue