Add new redistribute() directive (#20185)

Some packages can't be redistributed in source or binary form. We need an explicit way to say that in a package.

This adds a `redistribute()` directive so that package authors can write, e.g.:

```python
    redistribute(source=False, binary=False)
```

You can also do this conditionally with `when=`, as with other directives, e.g.:

```python
    # 12.0 and higher are proprietary
    redistribute(source=False, binary=False, when="@12.0:")

    # can't redistribute when we depend on some proprietary dependency
    redistribute(source=False, binary=False, when="^proprietary-dependency")
```


To prevent Spack from adding either their sources or binaries to public mirrors and build caches. You can still unconditionally add things *if* you run either:
* `spack mirror create --private`
* `spack buildcache push --private`

But the default behavior for build caches is not to include non-redistributable packages in either mirrors or build caches.  We have previously done this manually for our public buildcache, but with this we can start maintaining redistributability directly in packages.

Caveats: currently the default for `redistribute()` is `True` for both `source` and `binary`, and you can only set either of them to `False` via this directive.

- [x] add `redistribute()` directive
- [x] add `redistribute_source` and `redistribute_binary` class methods to `PackageBase`
- [x] add `--private` option to `spack mirror`
- [x] add `--private` option to `spack buildcache push`
- [x] test exclusion of packages from source mirror (both as a root and as a dependency)
- [x] test exclusion of packages from binary mirror (both as a root and as a dependency)

lib/spack/llnl/util/tty/colify.py

@@ -237,7 +237,6 @@ def transpose():
 def colified(
     elts: List[Any],
     cols: int = 0,
-    output: Optional[IO] = None,
     indent: int = 0,
     padding: int = 2,
     tty: Optional[bool] = None,

lib/spack/spack/cmd/buildcache.py

@@ -133,6 +133,11 @@ def setup_parser(subparser: argparse.ArgumentParser):
         help="when pushing to an OCI registry, tag an image containing all root specs and their "
         "runtime dependencies",
     )
+    push.add_argument(
+        "--private",
+        action="store_true",
+        help="for a private mirror, include non-redistributable packages",
+    )
     arguments.add_common_arguments(push, ["specs", "jobs"])
     push.set_defaults(func=push_fn)
 
@@ -367,6 +372,25 @@ def _make_pool() -> MaybePool:
         return NoPool()
 
 
+def _skip_no_redistribute_for_public(specs):
+    remaining_specs = list()
+    removed_specs = list()
+    for spec in specs:
+        if spec.package.redistribute_binary:
+            remaining_specs.append(spec)
+        else:
+            removed_specs.append(spec)
+    if removed_specs:
+        colified_output = tty.colify.colified(list(s.name for s in removed_specs), indent=4)
+        tty.debug(
+            "The following specs will not be added to the binary cache"
+            " because they cannot be redistributed:\n"
+            f"{colified_output}\n"
+            "You can use `--private` to include them."
+        )
+    return remaining_specs
+
+
 def push_fn(args):
     """create a binary package and push it to a mirror"""
     if args.spec_file:
@@ -417,6 +441,8 @@ def push_fn(args):
         root="package" in args.things_to_install,
         dependencies="dependencies" in args.things_to_install,
     )
+    if not args.private:
+        specs = _skip_no_redistribute_for_public(specs)
 
     # When pushing multiple specs, print the url once ahead of time, as well as how
     # many specs are being pushed.

lib/spack/spack/cmd/mirror.py

@@ -71,6 +71,11 @@ def setup_parser(subparser):
         help="the number of versions to fetch for each spec, choose 'all' to"
         " retrieve all versions of each package",
     )
+    create_parser.add_argument(
+        "--private",
+        action="store_true",
+        help="for a private mirror, include non-redistributable packages",
+    )
     arguments.add_common_arguments(create_parser, ["specs"])
     arguments.add_concretizer_args(create_parser)
 
@@ -359,7 +364,6 @@ def concrete_specs_from_user(args):
     specs = filter_externals(specs)
     specs = list(set(specs))
     specs.sort(key=lambda s: (s.name, s.version))
-    specs, _ = lang.stable_partition(specs, predicate_fn=not_excluded_fn(args))
     return specs
 
 
@@ -404,36 +408,50 @@ def concrete_specs_from_cli_or_file(args):
     return specs
 
 
-def not_excluded_fn(args):
-    """Return a predicate that evaluate to True if a spec was not explicitly
-    excluded by the user.
-    """
-    exclude_specs = []
-    if args.exclude_file:
-        exclude_specs.extend(specs_from_text_file(args.exclude_file, concretize=False))
-    if args.exclude_specs:
-        exclude_specs.extend(spack.cmd.parse_specs(str(args.exclude_specs).split()))
+class IncludeFilter:
+    def __init__(self, args):
+        self.exclude_specs = []
+        if args.exclude_file:
+            self.exclude_specs.extend(specs_from_text_file(args.exclude_file, concretize=False))
+        if args.exclude_specs:
+            self.exclude_specs.extend(spack.cmd.parse_specs(str(args.exclude_specs).split()))
+        self.private = args.private
 
-    def not_excluded(x):
-        return not any(x.satisfies(y) for y in exclude_specs)
+    def __call__(self, x):
+        return all([self._not_license_excluded(x), self._not_cmdline_excluded(x)])
 
-    return not_excluded
+    def _not_license_excluded(self, x):
+        """True if the spec is for a private mirror, or as long as the
+        package does not explicitly forbid redistributing source."""
+        if self.private:
+            return True
+        elif x.package_class.redistribute_source(x):
+            return True
+        else:
+            tty.debug(
+                "Skip adding {0} to mirror: the package.py file"
+                " indicates that a public mirror should not contain"
+                " it.".format(x.name)
+            )
+            return False
+
+    def _not_cmdline_excluded(self, x):
+        """True if a spec was not explicitly excluded by the user."""
+        return not any(x.satisfies(y) for y in self.exclude_specs)
 
 
-def concrete_specs_from_environment(selection_fn):
+def concrete_specs_from_environment():
     env = ev.active_environment()
     assert env, "an active environment is required"
     mirror_specs = env.all_specs()
     mirror_specs = filter_externals(mirror_specs)
-    mirror_specs, _ = lang.stable_partition(mirror_specs, predicate_fn=selection_fn)
     return mirror_specs
 
 
-def all_specs_with_all_versions(selection_fn):
+def all_specs_with_all_versions():
     specs = [spack.spec.Spec(n) for n in spack.repo.all_package_names()]
     mirror_specs = spack.mirror.get_all_versions(specs)
     mirror_specs.sort(key=lambda s: (s.name, s.version))
-    mirror_specs, _ = lang.stable_partition(mirror_specs, predicate_fn=selection_fn)
     return mirror_specs
 
 
@@ -454,12 +472,6 @@ def versions_per_spec(args):
     return num_versions
 
 
-def create_mirror_for_individual_specs(mirror_specs, path, skip_unstable_versions):
-    present, mirrored, error = spack.mirror.create(path, mirror_specs, skip_unstable_versions)
-    tty.msg("Summary for mirror in {}".format(path))
-    process_mirror_stats(present, mirrored, error)
-
-
 def process_mirror_stats(present, mirrored, error):
     p, m, e = len(present), len(mirrored), len(error)
     tty.msg(
@@ -505,30 +517,28 @@ def mirror_create(args):
     # When no directory is provided, the source dir is used
     path = args.directory or spack.caches.fetch_cache_location()
 
+    mirror_specs, mirror_fn = _specs_and_action(args)
+    mirror_fn(mirror_specs, path=path, skip_unstable_versions=args.skip_unstable_versions)
+
+
+def _specs_and_action(args):
+    include_fn = IncludeFilter(args)
+
     if args.all and not ev.active_environment():
-        create_mirror_for_all_specs(
-            path=path,
-            skip_unstable_versions=args.skip_unstable_versions,
-            selection_fn=not_excluded_fn(args),
-        )
-        return
+        mirror_specs = all_specs_with_all_versions()
+        mirror_fn = create_mirror_for_all_specs
+    elif args.all and ev.active_environment():
+        mirror_specs = concrete_specs_from_environment()
+        mirror_fn = create_mirror_for_individual_specs
+    else:
+        mirror_specs = concrete_specs_from_user(args)
+        mirror_fn = create_mirror_for_individual_specs
 
-    if args.all and ev.active_environment():
-        create_mirror_for_all_specs_inside_environment(
-            path=path,
-            skip_unstable_versions=args.skip_unstable_versions,
-            selection_fn=not_excluded_fn(args),
-        )
-        return
-
-    mirror_specs = concrete_specs_from_user(args)
-    create_mirror_for_individual_specs(
-        mirror_specs, path=path, skip_unstable_versions=args.skip_unstable_versions
-    )
+    mirror_specs, _ = lang.stable_partition(mirror_specs, predicate_fn=include_fn)
+    return mirror_specs, mirror_fn
 
 
-def create_mirror_for_all_specs(path, skip_unstable_versions, selection_fn):
-    mirror_specs = all_specs_with_all_versions(selection_fn=selection_fn)
+def create_mirror_for_all_specs(mirror_specs, path, skip_unstable_versions):
     mirror_cache, mirror_stats = spack.mirror.mirror_cache_and_stats(
         path, skip_unstable_versions=skip_unstable_versions
     )
@@ -540,11 +550,10 @@ def create_mirror_for_all_specs(path, skip_unstable_versions, selection_fn):
     process_mirror_stats(*mirror_stats.stats())
 
 
-def create_mirror_for_all_specs_inside_environment(path, skip_unstable_versions, selection_fn):
-    mirror_specs = concrete_specs_from_environment(selection_fn=selection_fn)
-    create_mirror_for_individual_specs(
-        mirror_specs, path=path, skip_unstable_versions=skip_unstable_versions
-    )
+def create_mirror_for_individual_specs(mirror_specs, path, skip_unstable_versions):
+    present, mirrored, error = spack.mirror.create(path, mirror_specs, skip_unstable_versions)
+    tty.msg("Summary for mirror in {}".format(path))
+    process_mirror_stats(present, mirrored, error)
 
 
 def mirror_destroy(args):

lib/spack/spack/directives.py

@@ -27,6 +27,7 @@ class OpenMpi(Package):
   * ``variant``
   * ``version``
   * ``requires``
+  * ``redistribute``
 
 """
 import collections
@@ -63,6 +64,7 @@ class OpenMpi(Package):
 __all__ = [
     "DirectiveError",
     "DirectiveMeta",
+    "DisableRedistribute",
     "version",
     "conflicts",
     "depends_on",
@@ -75,6 +77,7 @@ class OpenMpi(Package):
     "resource",
     "build_system",
     "requires",
+    "redistribute",
 ]
 
 #: These are variant names used by Spack internally; packages can't use them
@@ -598,6 +601,64 @@ def _execute_depends_on(pkg: "spack.package_base.PackageBase"):
     return _execute_depends_on
 
 
+#: Store whether a given Spec source/binary should not be redistributed.
+class DisableRedistribute:
+    def __init__(self, source, binary):
+        self.source = source
+        self.binary = binary
+
+
+@directive("disable_redistribute")
+def redistribute(source=None, binary=None, when: WhenType = None):
+    """Can be used inside a Package definition to declare that
+    the package source and/or compiled binaries should not be
+    redistributed.
+
+    By default, Packages allow source/binary distribution (i.e. in
+    mirrors). Because of this, and because overlapping enable/
+    disable specs are not allowed, this directive only allows users
+    to explicitly disable redistribution for specs.
+    """
+
+    return lambda pkg: _execute_redistribute(pkg, source, binary, when)
+
+
+def _execute_redistribute(
+    pkg: "spack.package_base.PackageBase", source=None, binary=None, when: WhenType = None
+):
+    if source is None and binary is None:
+        return
+    elif (source is True) or (binary is True):
+        raise DirectiveError(
+            "Source/binary distribution are true by default, they can only "
+            "be explicitly disabled."
+        )
+
+    if source is None:
+        source = True
+    if binary is None:
+        binary = True
+
+    when_spec = _make_when_spec(when)
+    if not when_spec:
+        return
+    if source is False:
+        max_constraint = spack.spec.Spec(f"{pkg.name}@{when_spec.versions}")
+        if not max_constraint.satisfies(when_spec):
+            raise DirectiveError("Source distribution can only be disabled for versions")
+
+    if when_spec in pkg.disable_redistribute:
+        disable = pkg.disable_redistribute[when_spec]
+        if not source:
+            disable.source = True
+        if not binary:
+            disable.binary = True
+    else:
+        pkg.disable_redistribute[when_spec] = DisableRedistribute(
+            source=not source, binary=not binary
+        )
+
+
 @directive(("extendees", "dependencies"))
 def extends(spec, when=None, type=("build", "run"), patches=None):
     """Same as depends_on, but also adds this package to the extendee list.

lib/spack/spack/package_base.py

@@ -468,7 +468,41 @@ def _names(when_indexed_dictionary):
     return sorted(all_names)
 
 
-class PackageBase(WindowsRPath, PackageViewMixin, metaclass=PackageMeta):
+class RedistributionMixin:
+    """Logic for determining whether a Package is source/binary
+    redistributable.
+    """
+
+    #: Store whether a given Spec source/binary should not be
+    #: redistributed.
+    disable_redistribute: Dict["spack.spec.Spec", "spack.directives.DisableRedistribute"]
+
+    # Source redistribution must be determined before concretization
+    # (because source mirrors work with un-concretized Specs).
+    @classmethod
+    def redistribute_source(cls, spec):
+        """Whether it should be possible to add the source of this
+        package to a Spack mirror.
+        """
+        for when_spec, disable_redistribute in cls.disable_redistribute.items():
+            if disable_redistribute.source and spec.satisfies(when_spec):
+                return False
+
+        return True
+
+    @property
+    def redistribute_binary(self):
+        """Whether it should be possible to create a binary out of an
+        installed instance of this package.
+        """
+        for when_spec, disable_redistribute in self.__class__.disable_redistribute.items():
+            if disable_redistribute.binary and self.spec.satisfies(when_spec):
+                return False
+
+        return True
+
+
+class PackageBase(WindowsRPath, PackageViewMixin, RedistributionMixin, metaclass=PackageMeta):
     """This is the superclass for all spack packages.
 
     ***The Package class***

lib/spack/spack/test/cmd/buildcache.py

@@ -446,3 +446,10 @@ def test_push_and_install_with_mirror_marked_unsigned_does_not_require_extra_fla
 
     spec.package.do_uninstall(force=True)
     spec.package.do_install(**kwargs)
+
+
+def test_skip_no_redistribute(mock_packages, config):
+    specs = list(Spec("no-redistribute-dependent").concretized().traverse())
+    filtered = spack.cmd.buildcache._skip_no_redistribute_for_public(specs)
+    assert not any(s.name == "no-redistribute" for s in filtered)
+    assert any(s.name == "no-redistribute-dependent" for s in filtered)

lib/spack/spack/test/cmd/mirror.py

@@ -88,6 +88,7 @@ def __init__(
         exclude_file=None,
         exclude_specs=None,
         directory=None,
+        private=False,
     ):
         self.specs = specs or []
         self.all = all
@@ -96,6 +97,7 @@ def __init__(
         self.dependencies = dependencies
         self.exclude_file = exclude_file
         self.exclude_specs = exclude_specs
+        self.private = private
         self.directory = directory
 
 
@@ -104,7 +106,7 @@ def test_exclude_specs(mock_packages, config):
         specs=["mpich"], versions_per_spec="all", exclude_specs="mpich@3.0.1:3.0.2 mpich@1.0"
     )
 
-    mirror_specs = spack.cmd.mirror.concrete_specs_from_user(args)
+    mirror_specs, _ = spack.cmd.mirror._specs_and_action(args)
     expected_include = set(
         spack.spec.Spec(x).concretized() for x in ["mpich@3.0.3", "mpich@3.0.4", "mpich@3.0"]
     )
@@ -113,6 +115,19 @@ def test_exclude_specs(mock_packages, config):
     assert not any(spec.satisfies(y) for spec in mirror_specs for y in expected_exclude)
 
 
+def test_exclude_specs_public_mirror(mock_packages, config):
+    args = MockMirrorArgs(
+        specs=["no-redistribute-dependent"],
+        versions_per_spec="all",
+        dependencies=True,
+        private=False,
+    )
+
+    mirror_specs, _ = spack.cmd.mirror._specs_and_action(args)
+    assert not any(s.name == "no-redistribute" for s in mirror_specs)
+    assert any(s.name == "no-redistribute-dependent" for s in mirror_specs)
+
+
 def test_exclude_file(mock_packages, tmpdir, config):
     exclude_path = os.path.join(str(tmpdir), "test-exclude.txt")
     with open(exclude_path, "w") as exclude_file:
@@ -125,7 +140,7 @@ def test_exclude_file(mock_packages, tmpdir, config):
 
     args = MockMirrorArgs(specs=["mpich"], versions_per_spec="all", exclude_file=exclude_path)
 
-    mirror_specs = spack.cmd.mirror.concrete_specs_from_user(args)
+    mirror_specs, _ = spack.cmd.mirror._specs_and_action(args)
     expected_include = set(
         spack.spec.Spec(x).concretized() for x in ["mpich@3.0.3", "mpich@3.0.4", "mpich@3.0"]
     )
@@ -262,11 +277,9 @@ def test_mirror_destroy(
 class TestMirrorCreate:
     @pytest.mark.regression("31736", "31985")
     def test_all_specs_with_all_versions_dont_concretize(self):
-        args = MockMirrorArgs(exclude_file=None, exclude_specs=None)
-        specs = spack.cmd.mirror.all_specs_with_all_versions(
-            selection_fn=spack.cmd.mirror.not_excluded_fn(args)
-        )
-        assert all(not s.concrete for s in specs)
+        args = MockMirrorArgs(all=True, exclude_file=None, exclude_specs=None)
+        mirror_specs, _ = spack.cmd.mirror._specs_and_action(args)
+        assert all(not s.concrete for s in mirror_specs)
 
     @pytest.mark.parametrize(
         "cli_args,error_str",
@@ -324,8 +337,8 @@ def test_error_conditions(self, cli_args, error_str):
         ],
     )
     def test_exclude_specs_from_user(self, cli_args, not_expected, config):
-        specs = spack.cmd.mirror.concrete_specs_from_user(MockMirrorArgs(**cli_args))
-        assert not any(s.satisfies(y) for s in specs for y in not_expected)
+        mirror_specs, _ = spack.cmd.mirror._specs_and_action(MockMirrorArgs(**cli_args))
+        assert not any(s.satisfies(y) for s in mirror_specs for y in not_expected)
 
     @pytest.mark.parametrize("abstract_specs", [("bowtie", "callpath")])
     def test_specs_from_cli_are_the_same_as_from_file(self, abstract_specs, config, tmpdir):

lib/spack/spack/test/conftest.py

@@ -1966,17 +1966,24 @@ def mock_modules_root(tmp_path, monkeypatch):
     monkeypatch.setattr(spack.modules.common, "root_path", fn)
 
 
+_repo_name_id = 0
+
+
 def create_test_repo(tmpdir, pkg_name_content_tuples):
+    global _repo_name_id
+
     repo_path = str(tmpdir)
     repo_yaml = tmpdir.join("repo.yaml")
     with open(str(repo_yaml), "w") as f:
         f.write(
-            """\
+            f"""\
 repo:
-  namespace: testcfgrequirements
+  namespace: testrepo{str(_repo_name_id)}
 """
         )
 
+    _repo_name_id += 1
+
     packages_dir = tmpdir.join("packages")
     for pkg_name, pkg_str in pkg_name_content_tuples:
         pkg_dir = packages_dir.ensure(pkg_name, dir=True)

lib/spack/spack/test/directives.py

@@ -10,6 +10,7 @@
 import spack.repo
 import spack.spec
 import spack.version
+from spack.test.conftest import create_test_repo
 
 
 def test_false_directives_do_not_exist(mock_packages):
@@ -142,3 +143,86 @@ def test_version_type_validation():
     # Try passing a bogus type; it's just that we want a nice error message
     with pytest.raises(spack.version.VersionError, match=msg):
         spack.directives._execute_version(package(name="python"), {})
+
+
+_pkgx = (
+    "x",
+    """\
+class X(Package):
+    version("1.3")
+    version("1.2")
+    version("1.1")
+    version("1.0")
+
+    variant("foo", default=False)
+
+    redistribute(binary=False, when="@1.1")
+    redistribute(binary=False, when="@1.0:1.2+foo")
+    redistribute(source=False, when="@1.0:1.2")
+""",
+)
+
+
+_pkgy = (
+    "y",
+    """\
+class Y(Package):
+    version("2.1")
+    version("2.0")
+
+    variant("bar", default=False)
+
+    redistribute(binary=False, source=False)
+""",
+)
+
+
+@pytest.fixture
+def _create_test_repo(tmpdir, mutable_config):
+    yield create_test_repo(tmpdir, [_pkgx, _pkgy])
+
+
+@pytest.fixture
+def test_repo(_create_test_repo, monkeypatch, mock_stage):
+    with spack.repo.use_repositories(_create_test_repo) as mock_repo_path:
+        yield mock_repo_path
+
+
+@pytest.mark.parametrize(
+    "spec_str,distribute_src,distribute_bin",
+    [
+        ("x@1.1~foo", False, False),
+        ("x@1.2+foo", False, False),
+        ("x@1.2~foo", False, True),
+        ("x@1.0~foo", False, True),
+        ("x@1.3+foo", True, True),
+        ("y@2.0", False, False),
+        ("y@2.1+bar", False, False),
+    ],
+)
+def test_redistribute_directive(test_repo, spec_str, distribute_src, distribute_bin):
+    spec = spack.spec.Spec(spec_str)
+    assert spec.package_class.redistribute_source(spec) == distribute_src
+    concretized_spec = spec.concretized()
+    assert concretized_spec.package.redistribute_binary == distribute_bin
+
+
+def test_redistribute_override_when():
+    """Allow a user to call `redistribute` twice to separately disable
+    source and binary distribution for the same when spec.
+
+    The second call should not undo the effect of the first.
+    """
+
+    class MockPackage:
+        name = "mock"
+        disable_redistribute = {}
+
+    cls = MockPackage
+    spack.directives._execute_redistribute(cls, source=False, when="@1.0")
+    spec_key = spack.directives._make_when_spec("@1.0")
+    assert not cls.disable_redistribute[spec_key].binary
+    assert cls.disable_redistribute[spec_key].source
+    spack.directives._execute_redistribute(cls, binary=False, when="@1.0")
+    assert cls.disable_redistribute[spec_key].binary
+    assert cls.disable_redistribute[spec_key].source

share/spack/spack-completion.bash

@@ -571,7 +571,7 @@ _spack_buildcache() {
 _spack_buildcache_push() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -f --force --allow-root -a --unsigned -u --signed --key -k --update-index --rebuild-index --spec-file --only --fail-fast --base-image --tag -t -j --jobs"
+        SPACK_COMPREPLY="-h --help -f --force --allow-root -a --unsigned -u --signed --key -k --update-index --rebuild-index --spec-file --only --fail-fast --base-image --tag -t --private -j --jobs"
     else
         _mirrors
     fi
@@ -580,7 +580,7 @@ _spack_buildcache_push() {
 _spack_buildcache_create() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -f --force --allow-root -a --unsigned -u --signed --key -k --update-index --rebuild-index --spec-file --only --fail-fast --base-image --tag -t -j --jobs"
+        SPACK_COMPREPLY="-h --help -f --force --allow-root -a --unsigned -u --signed --key -k --update-index --rebuild-index --spec-file --only --fail-fast --base-image --tag -t --private -j --jobs"
     else
         _mirrors
     fi
@@ -1428,7 +1428,7 @@ _spack_mirror() {
 _spack_mirror_create() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --exclude-file --exclude-specs --skip-unstable-versions -D --dependencies -n --versions-per-spec -U --fresh --reuse --reuse-deps --deprecated"
+        SPACK_COMPREPLY="-h --help -d --directory -a --all -f --file --exclude-file --exclude-specs --skip-unstable-versions -D --dependencies -n --versions-per-spec --private -U --fresh --reuse --reuse-deps --deprecated"
     else
         _all_packages
     fi

share/spack/spack-completion.fish

@@ -700,7 +700,7 @@ complete -c spack -n '__fish_spack_using_command buildcache' -s h -l help -f -a
 complete -c spack -n '__fish_spack_using_command buildcache' -s h -l help -d 'show this help message and exit'
 
 # spack buildcache push
-set -g __fish_spack_optspecs_spack_buildcache_push h/help f/force a/allow-root u/unsigned signed k/key= update-index spec-file= only= fail-fast base-image= t/tag= j/jobs=
+set -g __fish_spack_optspecs_spack_buildcache_push h/help f/force a/allow-root u/unsigned signed k/key= update-index spec-file= only= fail-fast base-image= t/tag= private j/jobs=
 complete -c spack -n '__fish_spack_using_command_pos_remainder 1 buildcache push' -f -k -a '(__fish_spack_specs)'
 complete -c spack -n '__fish_spack_using_command buildcache push' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command buildcache push' -s h -l help -d 'show this help message and exit'
@@ -726,11 +726,13 @@ complete -c spack -n '__fish_spack_using_command buildcache push' -l base-image
 complete -c spack -n '__fish_spack_using_command buildcache push' -l base-image -r -d 'specify the base image for the buildcache'
 complete -c spack -n '__fish_spack_using_command buildcache push' -l tag -s t -r -f -a tag
 complete -c spack -n '__fish_spack_using_command buildcache push' -l tag -s t -r -d 'when pushing to an OCI registry, tag an image containing all root specs and their runtime dependencies'
+complete -c spack -n '__fish_spack_using_command buildcache push' -l private -f -a private
+complete -c spack -n '__fish_spack_using_command buildcache push' -l private -d 'for a private mirror, include non-redistributable packages'
 complete -c spack -n '__fish_spack_using_command buildcache push' -s j -l jobs -r -f -a jobs
 complete -c spack -n '__fish_spack_using_command buildcache push' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
 
 # spack buildcache create
-set -g __fish_spack_optspecs_spack_buildcache_create h/help f/force a/allow-root u/unsigned signed k/key= update-index spec-file= only= fail-fast base-image= t/tag= j/jobs=
+set -g __fish_spack_optspecs_spack_buildcache_create h/help f/force a/allow-root u/unsigned signed k/key= update-index spec-file= only= fail-fast base-image= t/tag= private j/jobs=
 complete -c spack -n '__fish_spack_using_command_pos_remainder 1 buildcache create' -f -k -a '(__fish_spack_specs)'
 complete -c spack -n '__fish_spack_using_command buildcache create' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command buildcache create' -s h -l help -d 'show this help message and exit'
@@ -756,6 +758,8 @@ complete -c spack -n '__fish_spack_using_command buildcache create' -l base-imag
 complete -c spack -n '__fish_spack_using_command buildcache create' -l base-image -r -d 'specify the base image for the buildcache'
 complete -c spack -n '__fish_spack_using_command buildcache create' -l tag -s t -r -f -a tag
 complete -c spack -n '__fish_spack_using_command buildcache create' -l tag -s t -r -d 'when pushing to an OCI registry, tag an image containing all root specs and their runtime dependencies'
+complete -c spack -n '__fish_spack_using_command buildcache create' -l private -f -a private
+complete -c spack -n '__fish_spack_using_command buildcache create' -l private -d 'for a private mirror, include non-redistributable packages'
 complete -c spack -n '__fish_spack_using_command buildcache create' -s j -l jobs -r -f -a jobs
 complete -c spack -n '__fish_spack_using_command buildcache create' -s j -l jobs -r -d 'explicitly set number of parallel jobs'
 
@@ -2216,7 +2220,7 @@ complete -c spack -n '__fish_spack_using_command mirror' -s n -l no-checksum -f
 complete -c spack -n '__fish_spack_using_command mirror' -s n -l no-checksum -d 'do not use checksums to verify downloaded files (unsafe)'
 
 # spack mirror create
-set -g __fish_spack_optspecs_spack_mirror_create h/help d/directory= a/all f/file= exclude-file= exclude-specs= skip-unstable-versions D/dependencies n/versions-per-spec= U/fresh reuse reuse-deps deprecated
+set -g __fish_spack_optspecs_spack_mirror_create h/help d/directory= a/all f/file= exclude-file= exclude-specs= skip-unstable-versions D/dependencies n/versions-per-spec= private U/fresh reuse reuse-deps deprecated
 complete -c spack -n '__fish_spack_using_command_pos_remainder 0 mirror create' -f -k -a '(__fish_spack_specs)'
 complete -c spack -n '__fish_spack_using_command mirror create' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command mirror create' -s h -l help -d 'show this help message and exit'
@@ -2236,6 +2240,8 @@ complete -c spack -n '__fish_spack_using_command mirror create' -s D -l dependen
 complete -c spack -n '__fish_spack_using_command mirror create' -s D -l dependencies -d 'also fetch all dependencies'
 complete -c spack -n '__fish_spack_using_command mirror create' -s n -l versions-per-spec -r -f -a versions_per_spec
 complete -c spack -n '__fish_spack_using_command mirror create' -s n -l versions-per-spec -r -d 'the number of versions to fetch for each spec, choose \'all\' to retrieve all versions of each package'
+complete -c spack -n '__fish_spack_using_command mirror create' -l private -f -a private
+complete -c spack -n '__fish_spack_using_command mirror create' -l private -d 'for a private mirror, include non-redistributable packages'
 complete -c spack -n '__fish_spack_using_command mirror create' -s U -l fresh -f -a concretizer_reuse
 complete -c spack -n '__fish_spack_using_command mirror create' -s U -l fresh -d 'do not reuse installed deps; build newest configuration'
 complete -c spack -n '__fish_spack_using_command mirror create' -l reuse -f -a concretizer_reuse

var/spack/repos/builtin.mock/packages/no-redistribute-dependent/package.py

@@ -0,0 +1,23 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class NoRedistributeDependent(AutotoolsPackage):
+    """Package with one dependency on a package that should not be
+    redistributed"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/no-redistribute-dependent-1.0.tar.gz"
+
+    version("1.0", "0123456789abcdef0123456789abcdef")
+
+    depends_on("no-redistribute")
+
+    def install(self, spec, prefix):
+        # sanity_check_prefix requires something in the install directory
+        # Test requires overriding the one provided by `AutotoolsPackage`
+        mkdirp(prefix.bin)

var/spack/repos/builtin.mock/packages/no-redistribute/package.py

@@ -0,0 +1,23 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+class NoRedistribute(Package):
+    """Package which has source code that should not be added to a public
+    mirror"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/no-redistribute-1.0.tar.gz"
+
+    redistribute(source=False, binary=False)
+
+    version("1.0", "0123456789abcdef0123456789abcdef")
+
+    def install(self, spec, prefix):
+        # sanity_check_prefix requires something in the install directory
+        # Test requires overriding the one provided by `AutotoolsPackage`
+        mkdirp(prefix.bin)

var/spack/repos/builtin/packages/namd/package.py

@@ -20,6 +20,7 @@ class Namd(MakefilePackage, CudaPackage):
     url = "file://{0}/NAMD_2.12_Source.tar.gz".format(os.getcwd())
     git = "https://charm.cs.illinois.edu/gerrit/namd.git"
     manual_download = True
+    redistribute(source=False, binary=False)
 
     maintainers("jcphill")
 

var/spack/repos/builtin/packages/nvhpc/package.py

@@ -390,6 +390,8 @@ class Nvhpc(Package):
 
     skip_version_audit = ["platform=darwin"]
 
+    redistribute(source=False, binary=False)
+
     for ver, packages in _versions.items():
         key = "{0}-{1}".format(platform.system(), platform.machine())
         pkg = packages.get(key)