hlrs-software-stack/spack
7
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Mirrors: add option to exclude packages from "mirror create" (#14154)

Browse source 
* add an --exclude-file option to 'spack mirror create' which allows a user to specify a file of specs to exclude when creating a mirror. this is anticipated to be useful especially when using the '--all' option

* allow specifying number of versions when mirroring all packages

* when mirroring all specs within an environment, include dependencies of root specs

* add '--exclude-specs' option to allow user to specify that specs should be excluded on the command line

* add test for excluding specs
This commit is contained in:
Peter Scheibel 2020-06-03 17:43:51 -07:00 committed by GitHub
parent 0086c47546
commit 24775697f5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 99 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
lib/spack/spack/cmd/mirror.py
View file

@ -45,6 +45,15 @@ def setup_parser(subparser):
" (this requires significant time and space)")
create_parser.add_argument(
'-f', '--file', help="file with specs of packages to put in mirror")
create_parser.add_argument(
'--exclude-file',
help="specs which Spack should not try to add to a mirror"
" (listed in a file, one per line)")
create_parser.add_argument(
'--exclude-specs',
help="specs which Spack should not try to add to a mirror"
" (specified on command line)")
create_parser.add_argument(
'--skip-unstable-versions', action='store_true',
help="don't cache versions unless they identify a stable (unchanging)"
@ -232,9 +241,7 @@ def _read_specs_from_file(filename):
return specs
def mirror_create(args):
"""Create a directory to be used as a spack mirror, and fill it with
package archives."""
def _determine_specs_to_mirror(args):
if args.specs and args.all:
raise SpackError("Cannot specify specs on command line if you"
" chose to mirror all specs with '--all'")
@ -264,6 +271,7 @@ def mirror_create(args):
tty.die("Cannot pass specs on the command line with --file.")
specs = _read_specs_from_file(args.file)
env_specs = None
if not specs:
# If nothing is passed, use environment or all if no active env
if not args.all:
@ -273,12 +281,9 @@ def mirror_create(args):
env = ev.get_env(args, 'mirror')
if env:
mirror_specs = env.specs_by_hash.values()
env_specs = env.all_specs()
else:
specs = [Spec(n) for n in spack.repo.all_package_names()]
mirror_specs = spack.mirror.get_all_versions(specs)
mirror_specs.sort(
key=lambda s: (s.name, s.version))
else:
# If the user asked for dependencies, traverse spec DAG get them.
if args.dependencies:
@ -297,11 +302,38 @@ def mirror_create(args):
msg = 'Skipping {0} as it is an external spec.'
tty.msg(msg.format(spec.cshort_spec))
if env_specs:
if args.versions_per_spec:
tty.warn("Ignoring '--versions-per-spec' for mirroring specs"
" in environment.")
mirror_specs = env_specs
else:
if num_versions == 'all':
mirror_specs = spack.mirror.get_all_versions(specs)
else:
mirror_specs = spack.mirror.get_matching_versions(
specs, num_versions=num_versions)
mirror_specs.sort(
key=lambda s: (s.name, s.version))
exclude_specs = []
if args.exclude_file:
exclude_specs.extend(_read_specs_from_file(args.exclude_file))
if args.exclude_specs:
exclude_specs.extend(
spack.cmd.parse_specs(str(args.exclude_specs).split()))
if exclude_specs:
mirror_specs = list(
x for x in mirror_specs
if not any(x.satisfies(y, strict=True) for y in exclude_specs))
return mirror_specs
def mirror_create(args):
"""Create a directory to be used as a spack mirror, and fill it with
package archives."""
mirror_specs = _determine_specs_to_mirror(args)
mirror = spack.mirror.Mirror(
args.directory or spack.config.get('config:source_cache'))

25
lib/spack/spack/environment.py
View file

@ -1232,26 +1232,19 @@ def install_all(self, args=None):
self._install(spec, **kwargs)
def all_specs_by_hash(self):
"""Map of hashes to spec for all specs in this environment."""
# Note this uses dag-hashes calculated without build deps as keys,
# whereas the environment tracks specs based on dag-hashes calculated
# with all dependencies. This function should not be used by an
# Environment object for management of its own data structures
hashes = {}
for h in self.concretized_order:
specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
for spec in specs:
hashes[spec.dag_hash()] = spec
return hashes
def all_specs(self):
"""Return all specs, even those a user spec would shadow."""
return sorted(self.all_specs_by_hash().values())
all_specs = set()
for h in self.concretized_order:
all_specs.update(self.specs_by_hash[h].traverse())
return sorted(all_specs)
def all_hashes(self):
"""Return all specs, even those a user spec would shadow."""
return list(self.all_specs_by_hash().keys())
"""Return hashes of all specs.
Note these hashes exclude build dependencies."""
return list(set(s.dag_hash() for s in self.all_specs()))
def roots(self):
"""Specs explicitly requested by the user *in this environment*.

50
lib/spack/spack/test/cmd/mirror.py
View file

@ -89,6 +89,56 @@ def test_mirror_skip_unstable(tmpdir_factory, mock_packages, config,
set(['trivial-pkg-with-valid-hash']))
class MockMirrorArgs(object):
def __init__(self, specs=None, all=False, file=None,
versions_per_spec=None, dependencies=False,
exclude_file=None, exclude_specs=None):
self.specs = specs or []
self.all = all
self.file = file
self.versions_per_spec = versions_per_spec
self.dependencies = dependencies
self.exclude_file = exclude_file
self.exclude_specs = exclude_specs
def test_exclude_specs(mock_packages):
args = MockMirrorArgs(
specs=['mpich'],
versions_per_spec='all',
exclude_specs="mpich@3.0.1:3.0.2 mpich@1.0")
mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args)
expected_include = set(spack.spec.Spec(x) for x in
['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0'])
expected_exclude = set(spack.spec.Spec(x) for x in
['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0'])
assert expected_include <= set(mirror_specs)
assert (not expected_exclude & set(mirror_specs))
def test_exclude_file(mock_packages, tmpdir):
exclude_path = os.path.join(str(tmpdir), 'test-exclude.txt')
with open(exclude_path, 'w') as exclude_file:
exclude_file.write("""\
mpich@3.0.1:3.0.2
mpich@1.0
""")
args = MockMirrorArgs(
specs=['mpich'],
versions_per_spec='all',
exclude_file=exclude_path)
mirror_specs = spack.cmd.mirror._determine_specs_to_mirror(args)
expected_include = set(spack.spec.Spec(x) for x in
['mpich@3.0.3', 'mpich@3.0.4', 'mpich@3.0'])
expected_exclude = set(spack.spec.Spec(x) for x in
['mpich@3.0.1', 'mpich@3.0.2', 'mpich@1.0'])
assert expected_include <= set(mirror_specs)
assert (not expected_exclude & set(mirror_specs))
def test_mirror_crud(tmp_scope, capsys):
with capsys.disabled():
mirror('add', '--scope', tmp_scope, 'mirror', 'http://spack.io')

2
share/spack/spack-completion.bash
View file

@ -1042,7 +1042,7 @@ _spack_mirror() {
_spack_mirror_create() {
if $list_options
then
SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --skip-unstable-versions -D --dependencies -n --versions-per-spec"
SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --exclude-file --exclude-specs --skip-unstable-versions -D --dependencies -n --versions-per-spec"
else
_all_packages
fi