Allow choosing the name of the packages subdirectory in repositories (#36643)

Co-authored-by: becker33 <becker33@users.noreply.github.com>

lib/spack/docs/repositories.rst

@@ -32,11 +32,16 @@ A package repository a directory structured like this::
           ...
 
 The top-level ``repo.yaml`` file contains configuration metadata for the
-repository, and the ``packages`` directory contains subdirectories for
-each package in the repository.  Each package directory contains a
-``package.py`` file and any patches or other files needed to build the
+repository. The packages subdirectory, typically ``packages``, contains
+subdirectories for each package in the repository.  Each package directory
+contains a ``package.py`` file and any patches or other files needed to build the
 package.
 
+The ``repo.yaml`` file may also contain a ``subdirectory`` key,
+which can modify the name of the subdirectory used for packages. As seen above,
+the default value is ``packages``. An empty string (``subdirectory: ''``) requires
+a flattened repo structure in which the package names are top-level subdirectories.
+
 Package repositories allow you to:
 
 1. Maintain your own packages separately from Spack;
@@ -373,6 +378,24 @@ You can supply a custom namespace with a second argument, e.g.:
   repo:
     namespace: 'llnl.comp'
 
+You can also create repositories with custom structure with the ``-d/--subdirectory``
+argument, e.g.:
+
+.. code-block:: console
+
+  $ spack repo create -d applications myrepo apps
+  ==> Created repo with namespace 'apps'.
+  ==> To register it with Spack, run this command:
+    spack repo add ~/myrepo
+
+  $ ls myrepo
+  applications/  repo.yaml
+
+  $ cat myrepo/repo.yaml
+  repo:
+    namespace: apps
+    subdirectory: applications
+
 ^^^^^^^^^^^^^^^^^^
 ``spack repo add``
 ^^^^^^^^^^^^^^^^^^

lib/spack/spack/cmd/repo.py

@@ -32,6 +32,17 @@ def setup_parser(subparser):
         help="namespace to identify packages in the repository. " "defaults to the directory name",
         nargs="?",
     )
+    create_parser.add_argument(
+        "-d",
+        "--subdirectory",
+        action="store",
+        dest="subdir",
+        default=spack.repo.packages_dir_name,
+        help=(
+            "subdirectory to store packages in the repository."
+            " Default 'packages'. Use an empty string for no subdirectory."
+        ),
+    )
 
     # List
     list_parser = sp.add_parser("list", help=repo_list.__doc__)
@@ -70,7 +81,7 @@ def setup_parser(subparser):
 
 def repo_create(args):
     """Create a new package repository."""
-    full_path, namespace = spack.repo.create_repo(args.directory, args.namespace)
+    full_path, namespace = spack.repo.create_repo(args.directory, args.namespace, args.subdir)
     tty.msg("Created repo with namespace '%s'." % namespace)
     tty.msg("To register it with spack, run this command:", "spack repo add %s" % full_path)
 

lib/spack/spack/repo.py

@@ -935,12 +935,6 @@ def check(condition, msg):
         self.config_file = os.path.join(self.root, repo_config_name)
         check(os.path.isfile(self.config_file), "No %s found in '%s'" % (repo_config_name, root))
 
-        self.packages_path = os.path.join(self.root, packages_dir_name)
-        check(
-            os.path.isdir(self.packages_path),
-            "No directory '%s' found in '%s'" % (packages_dir_name, root),
-        )
-
         # Read configuration and validate namespace
         config = self._read_config()
         check(
@@ -961,6 +955,13 @@ def check(condition, msg):
         # Keep name components around for checking prefixes.
         self._names = self.full_namespace.split(".")
 
+        packages_dir = config.get("subdirectory", packages_dir_name)
+        self.packages_path = os.path.join(self.root, packages_dir)
+        check(
+            os.path.isdir(self.packages_path),
+            "No directory '%s' found in '%s'" % (packages_dir, root),
+        )
+
         # These are internal cache variables.
         self._modules = {}
         self._classes = {}
@@ -1150,7 +1151,7 @@ def all_package_names(self, include_virtuals=False):
 
     def package_path(self, name):
         """Get path to package.py file for this repo."""
-        return os.path.join(self.root, packages_dir_name, name, package_file_name)
+        return os.path.join(self.packages_path, name, package_file_name)
 
     def all_package_paths(self):
         for name in self.all_package_names():
@@ -1287,7 +1288,7 @@ def __contains__(self, pkg_name):
 RepoType = Union[Repo, RepoPath]
 
 
-def create_repo(root, namespace=None):
+def create_repo(root, namespace=None, subdir=packages_dir_name):
     """Create a new repository in root with the specified namespace.
 
     If the namespace is not provided, use basename of root.
@@ -1318,12 +1319,14 @@ def create_repo(root, namespace=None):
 
     try:
         config_path = os.path.join(root, repo_config_name)
-        packages_path = os.path.join(root, packages_dir_name)
+        packages_path = os.path.join(root, subdir)
 
         fs.mkdirp(packages_path)
         with open(config_path, "w") as config:
             config.write("repo:\n")
-            config.write("  namespace: '%s'\n" % namespace)
+            config.write(f"  namespace: '{namespace}'\n")
+            if subdir != packages_dir_name:
+                config.write(f"  subdirectory: '{subdir}'\n")
 
     except (IOError, OSError) as e:
         # try to clean up.

lib/spack/spack/test/repo.py

@@ -11,11 +11,11 @@
 import spack.repo
 
 
-@pytest.fixture()
-def extra_repo(tmpdir_factory):
+@pytest.fixture(params=["packages", "", "foo"])
+def extra_repo(tmpdir_factory, request):
     repo_namespace = "extra_test_repo"
     repo_dir = tmpdir_factory.mktemp(repo_namespace)
-    repo_dir.ensure("packages", dir=True)
+    repo_dir.ensure(request.param, dir=True)
 
     with open(str(repo_dir.join("repo.yaml")), "w") as f:
         f.write(
@@ -24,7 +24,9 @@ def extra_repo(tmpdir_factory):
   namespace: extra_test_repo
 """
         )
-    return spack.repo.Repo(str(repo_dir))
+        if request.param != "packages":
+            f.write(f"  subdirectory: '{request.param}'")
+    return (spack.repo.Repo(str(repo_dir)), request.param)
 
 
 def test_repo_getpkg(mutable_mock_repo):
@@ -33,13 +35,13 @@ def test_repo_getpkg(mutable_mock_repo):
 
 
 def test_repo_multi_getpkg(mutable_mock_repo, extra_repo):
-    mutable_mock_repo.put_first(extra_repo)
+    mutable_mock_repo.put_first(extra_repo[0])
     mutable_mock_repo.get_pkg_class("a")
     mutable_mock_repo.get_pkg_class("builtin.mock.a")
 
 
 def test_repo_multi_getpkgclass(mutable_mock_repo, extra_repo):
-    mutable_mock_repo.put_first(extra_repo)
+    mutable_mock_repo.put_first(extra_repo[0])
     mutable_mock_repo.get_pkg_class("a")
     mutable_mock_repo.get_pkg_class("builtin.mock.a")
 
@@ -63,9 +65,9 @@ def test_repo_last_mtime():
 
 
 def test_repo_invisibles(mutable_mock_repo, extra_repo):
-    with open(os.path.join(extra_repo.root, "packages", ".invisible"), "w"):
+    with open(os.path.join(extra_repo[0].root, extra_repo[1], ".invisible"), "w"):
         pass
-    extra_repo.all_package_names()
+    extra_repo[0].all_package_names()
 
 
 @pytest.mark.parametrize("attr_name,exists", [("cmake", True), ("__sphinx_mock__", False)])

share/spack/spack-completion.bash

@@ -1606,7 +1606,7 @@ _spack_repo() {
 _spack_repo_create() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help"
+        SPACK_COMPREPLY="-h --help -d --subdirectory"
     else
         _repos
     fi